From dbdc2ef95227771921848084bf4db28f91064907 Mon Sep 17 00:00:00 2001 From: luc Date: Fri, 28 Aug 2026 14:02:05 +0200 Subject: [PATCH] dataset-retrieval/retrieving-bluesky-datasets.ipynb: add --- .../retrieving-bluesky-datasets.ipynb | 508 ++++++++++++++++++ 1 file changed, 508 insertions(+) create mode 100644 dataset-retrieval/retrieving-bluesky-datasets.ipynb diff --git a/dataset-retrieval/retrieving-bluesky-datasets.ipynb b/dataset-retrieval/retrieving-bluesky-datasets.ipynb new file mode 100644 index 0000000..9a31618 --- /dev/null +++ b/dataset-retrieval/retrieving-bluesky-datasets.ipynb @@ -0,0 +1,508 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Retrieving Bluesky datasets\n", + "\n", + "Retrieving Bluesky datasets with the Bluesky API." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Retrieving Bluesky credentials and anonymity encryption key from credentials file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\".credentials\", \"r\") as file:\n", + " for line in file:\n", + " if 'bluesky_username' in line:\n", + " username = line.split('bluesky_username=')[1].strip()\n", + " break\n", + " for line in file:\n", + " if 'bluesky_password' in line:\n", + " password = line.split('bluesky_password=')[1].strip()\n", + " break" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Initializing the Bluesky API." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from atproto import Client\n", + "\n", + "# Initialize and log in\n", + "client = Client()\n", + "client.login(username, password);" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Converting the retrieval start and end time period to timestamps." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime, timezone, timedelta\n", + "\n", + "case = '...'\n", + "\n", + "start_date_string = '...'\n", + "end_date_string = '...'\n", + "\n", + "start_timestamp = datetime.strptime(start_date_string, '%Y-%m-%d').replace(tzinfo=timezone.utc)\n", + "end_timestamp = datetime.strptime(end_date_string, '%Y-%m-%d').replace(tzinfo=timezone.utc)\n", + "\n", + "iso_start_timestamp = start_timestamp.isoformat().replace('+00:00', 'Z')\n", + "iso_end_timestamp = end_timestamp.isoformat().replace('+00:00', 'Z')" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define functions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import re\n", + "from tqdm import tqdm\n", + "from more_itertools import chunked\n", + "\n", + "# Remove URLs, hashtags, mentions, emojis and whitespaces\n", + "def clean_text(text):\n", + " text = re.sub(r'http\\S+|www\\S+|https\\S+', '', text, flags=re.MULTILINE)\n", + " text = re.sub(r'#\\S+', '', text)\n", + " text = re.sub(r'@\\S+', '', text)\n", + " text = re.sub(r'[^\\w\\s,]', '', text)\n", + " text = re.sub(r'\\s+', ' ', text).strip()\n", + " return text\n", + "\n", + "# Search for posts containing a specific keyword\n", + "def get_posts_by_keyword(keyword, since, until, limit=300):\n", + " try:\n", + " num_posts = 0\n", + " post_data = []\n", + "\n", + " progress_bar = tqdm(total=limit, desc=\"Fetching posts\")\n", + "\n", + " while num_posts < limit:\n", + " \n", + " search_results = client.app.bsky.feed.search_posts({'q': keyword, 'since': since, 'until': until, 'lang': 'en', 'limit': 100})\n", + " posts = search_results.posts\n", + "\n", + " for post in posts:\n", + " # Extract post details\n", + " text = clean_text(post.record.text) if hasattr(post.record, 'text') else np.nan\n", + " iso_date = post.record.created_at if hasattr(post.record, 'created_at') else np.nan\n", + " user = post.author.handle if hasattr(post.author, 'handle') else np.nan\n", + "\n", + " post_id = post.uri if hasattr(post, 'uri') else np.nan\n", + " parent_id = post.record.reply.parent.uri if hasattr(post.record, 'reply') and hasattr(post.record.reply, 'parent') else 'no_parent'\n", + "\n", + " # Format date-time\n", + " if iso_date != np.nan:\n", + " date_obj = datetime.fromisoformat(iso_date.replace('Z', '+00:00'))\n", + " date_str = date_obj.strftime('%Y-%m-%d %H:%M:%S')\n", + " else:\n", + " date_str = np.nan\n", + "\n", + " if post_id != np.nan and user != np.nan:\n", + " post_data.append({\n", + " 'date': date_str,\n", + " 'text': text,\n", + " 'user': user,\n", + " 'id': post_id,\n", + " 'parentid': parent_id\n", + " })\n", + " else:\n", + " continue\n", + " \n", + " until = iso_date\n", + " num_posts += len(posts)\n", + " progress_bar.update(len(posts))\n", + " progress_bar.set_postfix_str(f'now at: {date_str}')\n", + "\n", + " if date_obj < (start_timestamp + timedelta(hours=1)):\n", + " break\n", + "\n", + " progress_bar.close()\n", + " df = pd.DataFrame(post_data)\n", + " return df\n", + "\n", + " except Exception as e:\n", + " print(f\"Error fetching posts: {e}\")\n", + " return pd.DataFrame()\n", + "\n", + "# Obtain posts from uri list\n", + "def get_posts(uris):\n", + " try:\n", + " post_data = []\n", + "\n", + " progress_bar = tqdm(total=len(uris), desc=\"Fetching posts\")\n", + "\n", + " for uribit in list(chunked(uris,20)):\n", + "\n", + " search_results = client.app.bsky.feed.get_posts({'uris': uribit})\n", + " posts = search_results.posts\n", + "\n", + " for post in posts: \n", + " # Extract post details\n", + " text = clean_text(post.record.text) if hasattr(post.record, 'text') else np.nan\n", + " iso_date = post.record.created_at if hasattr(post.record, 'created_at') else np.nan\n", + " user = post.author.handle if hasattr(post.author, 'handle') else np.nan\n", + "\n", + " post_id = post.uri if hasattr(post, 'uri') else np.nan\n", + " parent_id = post.record.reply.parent.uri if hasattr(post.record, 'reply') and hasattr(post.record.reply, 'parent') else 'no_parent'\n", + "\n", + " # Format date-time\n", + " if iso_date != np.nan:\n", + " date_obj = datetime.fromisoformat(iso_date.replace('Z', '+00:00'))\n", + " date_str = date_obj.strftime('%Y-%m-%d %H:%M:%S')\n", + " else:\n", + " date_str = np.nan\n", + "\n", + " if post_id != np.nan and user != np.nan:\n", + " post_data.append({\n", + " 'date': date_str,\n", + " 'text': text,\n", + " 'user': user,\n", + " 'id': post_id,\n", + " 'parentid': parent_id\n", + " })\n", + " else:\n", + " continue\n", + " \n", + " progress_bar.update(len(posts))\n", + " \n", + " progress_bar.close()\n", + " df = pd.DataFrame(post_data)\n", + " return df\n", + "\n", + " except Exception as e:\n", + " print(f\"Error fetching posts: {e}\")\n", + " return pd.DataFrame()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Retrieve posts with a keyword." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "raw_df_data = get_posts_by_keyword('...', iso_start_timestamp, iso_end_timestamp, 500000)\n", + "raw_df_data.info()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Retrieve absent posts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "id_list = raw_df_data['id'].to_list()\n", + "id_list.append('no_parent')\n", + "\n", + "absent_parents = raw_df_data.loc[~raw_df_data['parentid'].isin(id_list), 'parentid'].tolist()\n", + "\n", + "df_absent_data = get_posts(absent_parents)\n", + "\n", + "df_absent_data.info()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Combine dataframes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_data = pd.concat(raw_df_data, df_absent_data, ignore_index=True)\n", + "\n", + "df_data.info()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Drop duplicates." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_data = df_data.drop_duplicates()\n", + "df_data.info()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Remove empty entries." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_data.dropna(inplace=True,subset=['text'])\n", + "df_data.info()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Remove non-english entries." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from langdetect import detect\n", + "\n", + "def is_english(text):\n", + " try:\n", + " return detect(text) == 'en'\n", + " except:\n", + " return False\n", + " \n", + "df_data = df_data[df_data['text'].apply(is_english)].reset_index(drop=True)\n", + "df_data.info()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Exporting the dataframe to a csv dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_data.to_csv(f'../datasets/{case}.csv', index=False)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Count the number of posts per user. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "user_posts = df_data.groupby('user')['id'].count()\n", + "user_posts.describe()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Filter on social users." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "social_users = user_posts[(user_posts >= 5) & (user_posts <= 10)].index\n", + "df_reduced_data = df_data[df_data['user'].isin(social_users)]\n", + "df_reduced_data.info()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Close reduced dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "closed_df_reduced_data = df_reduced_data.copy()\n", + "\n", + "id_list = closed_df_reduced_data['id'].to_list()\n", + "id_list.append('no_parent')\n", + "\n", + "closed_df_reduced_data.loc[~closed_df_reduced_data['parentid'].isin(id_list), 'parentid'] = 'no_parent'\n", + "\n", + "closed_df_reduced_data.info()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Close dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "closed_df_data = df_data.copy()\n", + "\n", + "id_list = closed_df_data['id'].to_list()\n", + "id_list.append('no_parent')\n", + "\n", + "closed_df_data.loc[~closed_df_data['parentid'].isin(id_list), 'parentid'] = 'no_parent'\n", + "\n", + "closed_df_data.info()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Exporting the closed reduced dataframe to a csv dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "closed_df_reduced_data.to_csv(f'../datasets/{case}-social-closed.csv', index=False)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Exporting the closed dataframe to a csv dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "closed_df_data.to_csv(f'../datasets/{case}-closed.csv', index=False)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.6" + }, + "vscode": { + "interpreter": { + "hash": "8c75c0fdd1a718867cdcb84b32adcfdbeaad00b3a4e00a59385211aeed084d4c" + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}