{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Retrieving Bluesky datasets\n",
    "\n",
    "Retrieving Bluesky datasets with the Bluesky API."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Retrieving Bluesky credentials and anonymity encryption key from credentials file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\".credentials\", \"r\") as file:\n",
    "    for line in file:\n",
    "        if 'bluesky_username' in line:\n",
    "            username = line.split('bluesky_username=')[1].strip()\n",
    "            break\n",
    "    for line in file:\n",
    "        if 'bluesky_password' in line:\n",
    "            password = line.split('bluesky_password=')[1].strip()\n",
    "            break"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initializing the Bluesky API."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from atproto import Client\n",
    "\n",
    "# Initialize and log in\n",
    "client = Client()\n",
    "client.login(username, password);"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Converting the retrieval start and end time period to timestamps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime, timezone, timedelta\n",
    "\n",
    "case = '...'\n",
    "\n",
    "start_date_string = '...'\n",
    "end_date_string = '...'\n",
    "\n",
    "start_timestamp = datetime.strptime(start_date_string, '%Y-%m-%d').replace(tzinfo=timezone.utc)\n",
    "end_timestamp = datetime.strptime(end_date_string, '%Y-%m-%d').replace(tzinfo=timezone.utc)\n",
    "\n",
    "iso_start_timestamp = start_timestamp.isoformat().replace('+00:00', 'Z')\n",
    "iso_end_timestamp = end_timestamp.isoformat().replace('+00:00', 'Z')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "from tqdm import tqdm\n",
    "from more_itertools import chunked\n",
    "\n",
    "# Remove URLs, hashtags, mentions, emojis and whitespaces\n",
    "def clean_text(text):\n",
    "    text = re.sub(r'http\\S+|www\\S+|https\\S+', '', text, flags=re.MULTILINE)\n",
    "    text = re.sub(r'#\\S+', '', text)\n",
    "    text = re.sub(r'@\\S+', '', text)\n",
    "    text = re.sub(r'[^\\w\\s,]', '', text)\n",
    "    text = re.sub(r'\\s+', ' ', text).strip()\n",
    "    return text\n",
    "\n",
    "# Search for posts containing a specific keyword\n",
    "def get_posts_by_keyword(keyword, since, until, limit=300):\n",
    "    try:\n",
    "        num_posts = 0\n",
    "        post_data = []\n",
    "\n",
    "        progress_bar = tqdm(total=limit, desc=\"Fetching posts\")\n",
    "\n",
    "        while num_posts < limit:\n",
    "            \n",
    "            search_results = client.app.bsky.feed.search_posts({'q': keyword, 'since': since, 'until': until, 'lang': 'en', 'limit': 100})\n",
    "            posts = search_results.posts\n",
    "\n",
    "            for post in posts:\n",
    "                        # Extract post details\n",
    "                        text = clean_text(post.record.text) if hasattr(post.record, 'text') else np.nan\n",
    "                        iso_date = post.record.created_at if hasattr(post.record, 'created_at') else np.nan\n",
    "                        user = post.author.handle if hasattr(post.author, 'handle') else np.nan\n",
    "\n",
    "                        post_id = post.uri if hasattr(post, 'uri') else np.nan\n",
    "                        parent_id = post.record.reply.parent.uri if hasattr(post.record, 'reply') and hasattr(post.record.reply, 'parent') else 'no_parent'\n",
    "\n",
    "                        # Format date-time\n",
    "                        if iso_date != np.nan:\n",
    "                            date_obj = datetime.fromisoformat(iso_date.replace('Z', '+00:00'))\n",
    "                            date_str = date_obj.strftime('%Y-%m-%d %H:%M:%S')\n",
    "                        else:\n",
    "                            date_str = np.nan\n",
    "\n",
    "                        if post_id != np.nan and user != np.nan:\n",
    "                            post_data.append({\n",
    "                                'date': date_str,\n",
    "                                'text': text,\n",
    "                                'user': user,\n",
    "                                'id': post_id,\n",
    "                                'parentid': parent_id\n",
    "                            })\n",
    "                        else:\n",
    "                            continue\n",
    "                        \n",
    "            until = iso_date\n",
    "            num_posts += len(posts)\n",
    "            progress_bar.update(len(posts))\n",
    "            progress_bar.set_postfix_str(f'now at: {date_str}')\n",
    "\n",
    "            if date_obj < (start_timestamp + timedelta(hours=1)):\n",
    "                break\n",
    "\n",
    "        progress_bar.close()\n",
    "        df = pd.DataFrame(post_data)\n",
    "        return df\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"Error fetching posts: {e}\")\n",
    "        return pd.DataFrame()\n",
    "\n",
    "# Obtain posts from uri list\n",
    "def get_posts(uris):\n",
    "    try:\n",
    "        post_data = []\n",
    "\n",
    "        progress_bar = tqdm(total=len(uris), desc=\"Fetching posts\")\n",
    "\n",
    "        for uribit in list(chunked(uris,20)):\n",
    "\n",
    "            search_results = client.app.bsky.feed.get_posts({'uris': uribit})\n",
    "            posts = search_results.posts\n",
    "\n",
    "            for post in posts: \n",
    "                # Extract post details\n",
    "                text = clean_text(post.record.text) if hasattr(post.record, 'text') else np.nan\n",
    "                iso_date = post.record.created_at if hasattr(post.record, 'created_at') else np.nan\n",
    "                user = post.author.handle if hasattr(post.author, 'handle') else np.nan\n",
    "\n",
    "                post_id = post.uri if hasattr(post, 'uri') else np.nan\n",
    "                parent_id = post.record.reply.parent.uri if hasattr(post.record, 'reply') and hasattr(post.record.reply, 'parent') else 'no_parent'\n",
    "\n",
    "                # Format date-time\n",
    "                if iso_date != np.nan:\n",
    "                    date_obj = datetime.fromisoformat(iso_date.replace('Z', '+00:00'))\n",
    "                    date_str = date_obj.strftime('%Y-%m-%d %H:%M:%S')\n",
    "                else:\n",
    "                    date_str = np.nan\n",
    "\n",
    "                if post_id != np.nan and user != np.nan:\n",
    "                    post_data.append({\n",
    "                        'date': date_str,\n",
    "                        'text': text,\n",
    "                        'user': user,\n",
    "                        'id': post_id,\n",
    "                        'parentid': parent_id\n",
    "                    })\n",
    "                else:\n",
    "                    continue\n",
    "        \n",
    "            progress_bar.update(len(posts))\n",
    "    \n",
    "        progress_bar.close()\n",
    "        df = pd.DataFrame(post_data)\n",
    "        return df\n",
    "\n",
    "    except Exception as e:\n",
    "        print(f\"Error fetching posts: {e}\")\n",
    "        return pd.DataFrame()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Retrieve posts with a keyword."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_df_data = get_posts_by_keyword('...', iso_start_timestamp, iso_end_timestamp, 500000)\n",
    "raw_df_data.info()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Retrieve absent posts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_list = raw_df_data['id'].to_list()\n",
    "id_list.append('no_parent')\n",
    "\n",
    "absent_parents = raw_df_data.loc[~raw_df_data['parentid'].isin(id_list), 'parentid'].tolist()\n",
    "\n",
    "df_absent_data = get_posts(absent_parents)\n",
    "\n",
    "df_absent_data.info()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Combine dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data = pd.concat(raw_df_data, df_absent_data, ignore_index=True)\n",
    "\n",
    "df_data.info()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Drop duplicates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data = df_data.drop_duplicates()\n",
    "df_data.info()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove empty entries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data.dropna(inplace=True,subset=['text'])\n",
    "df_data.info()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove non-english entries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langdetect import detect\n",
    "\n",
    "def is_english(text):\n",
    "    try:\n",
    "        return detect(text) == 'en'\n",
    "    except:\n",
    "        return False\n",
    "    \n",
    "df_data = df_data[df_data['text'].apply(is_english)].reset_index(drop=True)\n",
    "df_data.info()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Exporting the dataframe to a csv dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data.to_csv(f'../datasets/{case}.csv', index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Count the number of posts per user. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_posts = df_data.groupby('user')['id'].count()\n",
    "user_posts.describe()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filter on social users."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "social_users = user_posts[(user_posts >= 5) & (user_posts <= 10)].index\n",
    "df_reduced_data = df_data[df_data['user'].isin(social_users)]\n",
    "df_reduced_data.info()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Close reduced dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "closed_df_reduced_data = df_reduced_data.copy()\n",
    "\n",
    "id_list = closed_df_reduced_data['id'].to_list()\n",
    "id_list.append('no_parent')\n",
    "\n",
    "closed_df_reduced_data.loc[~closed_df_reduced_data['parentid'].isin(id_list), 'parentid'] = 'no_parent'\n",
    "\n",
    "closed_df_reduced_data.info()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Close dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "closed_df_data = df_data.copy()\n",
    "\n",
    "id_list = closed_df_data['id'].to_list()\n",
    "id_list.append('no_parent')\n",
    "\n",
    "closed_df_data.loc[~closed_df_data['parentid'].isin(id_list), 'parentid'] = 'no_parent'\n",
    "\n",
    "closed_df_data.info()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Exporting the closed reduced dataframe to a csv dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "closed_df_reduced_data.to_csv(f'../datasets/{case}-social-closed.csv', index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Exporting the closed dataframe to a csv dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "closed_df_data.to_csv(f'../datasets/{case}-closed.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.14.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "8c75c0fdd1a718867cdcb84b32adcfdbeaad00b3a4e00a59385211aeed084d4c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}