532 lines
15 KiB
Text
532 lines
15 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Retrieving Bluesky datasets\n",
|
|
"\n",
|
|
"Retrieving Bluesky datasets with the Bluesky API."
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Retrieving Bluesky credentials and anonymity encryption key from credentials file."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"with open(\".credentials\", \"r\") as file:\n",
|
|
" for line in file:\n",
|
|
" if 'bluesky_username' in line:\n",
|
|
" username = line.split('bluesky_username=')[1].strip()\n",
|
|
" break\n",
|
|
" for line in file:\n",
|
|
" if 'bluesky_password' in line:\n",
|
|
" password = line.split('bluesky_password=')[1].strip()\n",
|
|
" break"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Initializing the Bluesky API."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from atproto import Client\n",
|
|
"\n",
|
|
"# Initialize and log in\n",
|
|
"client = Client()\n",
|
|
"client.login(username, password);"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Converting the retrieval start and end time period to timestamps."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from datetime import datetime, timezone, timedelta\n",
|
|
"\n",
|
|
"case = '...'\n",
|
|
"\n",
|
|
"start_date_string = '...'\n",
|
|
"end_date_string = '...'\n",
|
|
"\n",
|
|
"start_timestamp = datetime.strptime(start_date_string, '%Y-%m-%d').replace(tzinfo=timezone.utc)\n",
|
|
"end_timestamp = datetime.strptime(end_date_string, '%Y-%m-%d').replace(tzinfo=timezone.utc)\n",
|
|
"\n",
|
|
"iso_start_timestamp = start_timestamp.isoformat().replace('+00:00', 'Z')\n",
|
|
"iso_end_timestamp = end_timestamp.isoformat().replace('+00:00', 'Z')"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Define functions."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import re, random, time\n",
|
|
"import pandas as pd\n",
|
|
"import numpy as np\n",
|
|
"from tqdm import tqdm\n",
|
|
"from more_itertools import chunked\n",
|
|
"\n",
|
|
"# Remove URLs, hashtags, mentions, emojis and whitespaces\n",
|
|
"def clean_text(text):\n",
|
|
" if not isinstance(text, str):\n",
|
|
" return np.nan\n",
|
|
" text = re.sub(r'http\\S+|www\\S+|https\\S+', '', text, flags=re.MULTILINE)\n",
|
|
" text = re.sub(r'#\\S+', '', text)\n",
|
|
" text = re.sub(r'@\\S+', '', text)\n",
|
|
" text = re.sub(r'[^\\w\\s,]', '', text)\n",
|
|
" text = re.sub(r'\\s+', ' ', text).strip()\n",
|
|
" return text\n",
|
|
"\n",
|
|
"# Retry helper\n",
|
|
"def call_with_retry(fn, *args, max_retries=5, base_delay=2, **kwargs):\n",
|
|
" last_exc = None\n",
|
|
" for attempt in range(1, max_retries + 1):\n",
|
|
" try:\n",
|
|
" return fn(*args, **kwargs)\n",
|
|
" except Exception as e:\n",
|
|
" last_exc = e\n",
|
|
" wait = base_delay * (2 ** (attempt - 1)) + random.uniform(0, 1)\n",
|
|
" print(f' [retry {attempt}/{max_retries}] {type(e).__name__}: {e} - retrying in {wait:.1f}s')\n",
|
|
" time.sleep(wait)\n",
|
|
" raise last_exc\n",
|
|
"\n",
|
|
"# Post field extraction\n",
|
|
"def extract_post(post):\n",
|
|
" try:\n",
|
|
" text = clean_text(post.record.text) if hasattr(post.record, 'text') else None\n",
|
|
" iso_date = post.record.created_at if hasattr(post.record, 'created_at') else None\n",
|
|
" user = post.author.handle if hasattr(post.author, 'handle') else None\n",
|
|
" post_id = post.uri if hasattr(post, 'uri') else None\n",
|
|
" parent_id = (\n",
|
|
" post.record.reply.parent.uri\n",
|
|
" if hasattr(post.record, 'reply') and hasattr(post.record.reply, 'parent')\n",
|
|
" else 'no_parent'\n",
|
|
" )\n",
|
|
" \n",
|
|
" if iso_date is None or post_id is None or user is None:\n",
|
|
" return None\n",
|
|
" \n",
|
|
" date_obj = datetime.fromisoformat(iso_date.replace('Z', '+00:00'))\n",
|
|
" date_str = date_obj.strftime('%Y-%m-%d %H:%M:%S')\n",
|
|
" \n",
|
|
" return {\n",
|
|
" 'date': date_str,\n",
|
|
" 'text': text if text is not None else np.nan,\n",
|
|
" 'user': user,\n",
|
|
" 'id': post_id,\n",
|
|
" 'parentid': parent_id,\n",
|
|
" }, date_obj\n",
|
|
" except Exception as e:\n",
|
|
" print(f\" Skipping malformed post: {e}\")\n",
|
|
" return None\n",
|
|
"\n",
|
|
"# Cleanup\n",
|
|
"def _dedupe(rows):\n",
|
|
" if not rows:\n",
|
|
" return pd.DataFrame()\n",
|
|
" df = pd.DataFrame(rows)\n",
|
|
" before = len(df)\n",
|
|
" df = df.drop_duplicates(subset='id')\n",
|
|
" df = df.dropna(subset=['text', 'id', 'user'])\n",
|
|
" df = df[df['text'].astype(str).str.strip() != '']\n",
|
|
" after = len(df)\n",
|
|
" if before != after:\n",
|
|
" print(f\"(Filtered {before - after} duplicate/empty rows out of {before}.)\")\n",
|
|
" return df.reset_index(drop=True)\n",
|
|
"\n",
|
|
" \n",
|
|
"# Search for posts containing a specific keyword\n",
|
|
"def get_posts_by_keyword(keyword, since, until, limit=300, page_size=100, max_retries=5):\n",
|
|
"\n",
|
|
" state = _STATE.setdefault(keyword, {'until': until, 'rows': []})\n",
|
|
" \n",
|
|
" if state['until'] != until and state['rows']:\n",
|
|
" print(f\"Resuming '{keyword}' from until={state['until']}, {len(state['rows'])} posts already fetched this session.\")\n",
|
|
" \n",
|
|
" since_dt = datetime.fromisoformat(since.replace('Z', '+00:00'))\n",
|
|
" progress_bar = tqdm(total=limit, initial=min(len(state['rows']), limit), desc=\"Fetching posts\")\n",
|
|
"\n",
|
|
" try:\n",
|
|
" while len(state['rows']) < limit:\n",
|
|
" try:\n",
|
|
" search_results = call_with_retry(client.app.bsky.feed.search_posts, {'q': keyword, 'since': since, 'until': state['until'], 'lang': 'en', 'limit': page_size}, max_retries=max_retries)\n",
|
|
" except Exception as e:\n",
|
|
" print(f\"Giving up on this page after {max_retries} retries: {e}\")\n",
|
|
" print(\"Progress so far is kept — just call this again to continue.\")\n",
|
|
" break\n",
|
|
" \n",
|
|
" posts = search_results.posts\n",
|
|
" if not posts:\n",
|
|
" print(\"No more posts returned — reached the end of available results.\")\n",
|
|
" break\n",
|
|
" \n",
|
|
" oldest_date_obj = None\n",
|
|
" for post in posts:\n",
|
|
" result = extract_post(post)\n",
|
|
" if result is None:\n",
|
|
" continue\n",
|
|
" row, date_obj = result\n",
|
|
" state['rows'].append(row)\n",
|
|
" if oldest_date_obj is None or date_obj < oldest_date_obj:\n",
|
|
" oldest_date_obj = date_obj\n",
|
|
" \n",
|
|
" progress_bar.update(len(posts))\n",
|
|
" \n",
|
|
" if oldest_date_obj is not None:\n",
|
|
" state['until'] = oldest_date_obj.isoformat().replace('+00:00', 'Z')\n",
|
|
" progress_bar.set_postfix_str(f'now at: {state[\"until\"]}')\n",
|
|
" \n",
|
|
" if oldest_date_obj < since_dt + timedelta(hours=1):\n",
|
|
" print(\"Reached the `since` boundary.\")\n",
|
|
" break\n",
|
|
" \n",
|
|
" finally:\n",
|
|
" progress_bar.close()\n",
|
|
" \n",
|
|
" print(f\"{len(state['rows'])} posts fetched in total this session for '{keyword}'. Call again with the same checkpoint_name to keep going.\")\n",
|
|
" return _dedupe(state['rows'])\n",
|
|
"\n",
|
|
"# Obtain posts from uri list\n",
|
|
"def get_posts(uris, chunk_size=20, max_retries=5):\n",
|
|
" state = _STATE.setdefault('_by_uri', {'done_uris': set(), 'rows': []})\n",
|
|
" \n",
|
|
" remaining = [u for u in uris if u not in state['done_uris']]\n",
|
|
" chunks = list(chunked(remaining, chunk_size))\n",
|
|
" \n",
|
|
" progress_bar = tqdm(total=len(uris), initial=len(state['done_uris']), desc=\"Fetching posts\")\n",
|
|
" \n",
|
|
" try:\n",
|
|
" for uribit in chunks:\n",
|
|
" try:\n",
|
|
" search_results = call_with_retry(client.app.bsky.feed.get_posts, {'uris': uribit}, max_retries=max_retries)\n",
|
|
" except Exception as e:\n",
|
|
" print(f\"Skipping this chunk of {len(uribit)} URIs after {max_retries} retries: {e}\")\n",
|
|
" continue\n",
|
|
" \n",
|
|
" for post in search_results.posts:\n",
|
|
" result = extract_post(post)\n",
|
|
" if result is None:\n",
|
|
" continue\n",
|
|
" row, _ = result\n",
|
|
" state['rows'].append(row)\n",
|
|
" \n",
|
|
" state['done_uris'].update(uribit)\n",
|
|
" progress_bar.update(len(uribit))\n",
|
|
" \n",
|
|
" finally:\n",
|
|
" progress_bar.close()\n",
|
|
" \n",
|
|
" print(f\"{len(state['done_uris'])}/{len(uris)} URIs processed in total.\")\n",
|
|
" return _dedupe(state['rows'])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Define state"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"_STATE = {}"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Retrieve posts with a keyword."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"raw_df_data = get_posts_by_keyword('...', iso_start_timestamp, iso_end_timestamp, 100000)\n",
|
|
"raw_df_data.info()"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Retrieve absent posts."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"_STATE = {}\n",
|
|
"id_list = raw_df_data['id'].to_list()\n",
|
|
"id_list.append('no_parent')\n",
|
|
"absent_parents = raw_df_data.loc[~raw_df_data['parentid'].isin(id_list), 'parentid'].tolist()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_absent_data = get_posts(absent_parents)\n",
|
|
"df_absent_data.info()"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Combine dataframes"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_data = pd.concat([raw_df_data, df_absent_data], ignore_index=True)\n",
|
|
"df_data.info()"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Remove non-english entries."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from langdetect import detect\n",
|
|
"\n",
|
|
"def is_english(text):\n",
|
|
" try:\n",
|
|
" return detect(text) == 'en'\n",
|
|
" except:\n",
|
|
" return False\n",
|
|
" \n",
|
|
"df_data = df_data[df_data['text'].apply(is_english)].reset_index(drop=True)\n",
|
|
"df_data.info()"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Exporting the dataframe to a csv dataset."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"df_data.to_csv(f'{case}.csv', index=False)"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Count the number of posts per user. "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"user_posts = df_data.groupby('user')['id'].count()\n",
|
|
"user_posts.describe()"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Filter on social users."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"social_users = user_posts[(user_posts >= 5) & (user_posts <= 10)].index\n",
|
|
"df_reduced_data = df_data[df_data['user'].isin(social_users)]\n",
|
|
"df_reduced_data.info()"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Close reduced dataset."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"closed_df_reduced_data = df_reduced_data.copy()\n",
|
|
"\n",
|
|
"id_list = closed_df_reduced_data['id'].to_list()\n",
|
|
"id_list.append('no_parent')\n",
|
|
"\n",
|
|
"closed_df_reduced_data.loc[~closed_df_reduced_data['parentid'].isin(id_list), 'parentid'] = 'no_parent'\n",
|
|
"\n",
|
|
"closed_df_reduced_data.info()"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Close dataset."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"closed_df_data = df_data.copy()\n",
|
|
"\n",
|
|
"id_list = closed_df_data['id'].to_list()\n",
|
|
"id_list.append('no_parent')\n",
|
|
"\n",
|
|
"closed_df_data.loc[~closed_df_data['parentid'].isin(id_list), 'parentid'] = 'no_parent'\n",
|
|
"\n",
|
|
"closed_df_data.info()"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Exporting the closed reduced dataframe to a csv dataset."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"closed_df_reduced_data.to_csv(f'{case}-social-closed.csv', index=False)"
|
|
]
|
|
},
|
|
{
|
|
"attachments": {},
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Exporting the closed dataframe to a csv dataset."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"closed_df_data.to_csv(f'{case}-closed.csv', index=False)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3 (ipykernel)",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.14.6"
|
|
},
|
|
"vscode": {
|
|
"interpreter": {
|
|
"hash": "8c75c0fdd1a718867cdcb84b32adcfdbeaad00b3a4e00a59385211aeed084d4c"
|
|
}
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 4
|
|
}
|