physics/social-graphs
1
0
Fork 0
Code Issues Pull requests Activity

Compare commits

base: physics:8ac5aa2cb3bce0d58ecb724cea0fa0b3157ba52c
Branches Tags
physics:main
..
compare: physics:3c7e3fc018df2b74648ea40592d16540a62d2df4
Branches Tags
physics:main

No commits in common. "8ac5aa2cb3bce0d58ecb724cea0fa0b3157ba52c" and "3c7e3fc018df2b74648ea40592d16540a62d2df4" have entirely different histories.
8ac5aa2cb3 ... 3c7e3fc018

5 changed files with 182 additions and 274 deletions
Download patch file Download diff file Expand all files Collapse all files

296
dataset-retrieval/retrieving-bluesky-datasets.ipynb
View file

@ -98,16 +98,14 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"import re, random, time\n",
"import pandas as pd\n", "import pandas as pd\n",
"import numpy as np\n", "import numpy as np\n",
"import re\n",
"from tqdm import tqdm\n", "from tqdm import tqdm\n",
"from more_itertools import chunked\n", "from more_itertools import chunked\n",
"\n", "\n",
"# Remove URLs, hashtags, mentions, emojis and whitespaces\n", "# Remove URLs, hashtags, mentions, emojis and whitespaces\n",
"def clean_text(text):\n", "def clean_text(text):\n",
" if not isinstance(text, str):\n",
" return np.nan\n",
" text = re.sub(r'http\\S+|www\\S+|https\\S+', '', text, flags=re.MULTILINE)\n", " text = re.sub(r'http\\S+|www\\S+|https\\S+', '', text, flags=re.MULTILINE)\n",
" text = re.sub(r'#\\S+', '', text)\n", " text = re.sub(r'#\\S+', '', text)\n",
" text = re.sub(r'@\\S+', '', text)\n", " text = re.sub(r'@\\S+', '', text)\n",
@ -115,163 +113,110 @@
" text = re.sub(r'\\s+', ' ', text).strip()\n", " text = re.sub(r'\\s+', ' ', text).strip()\n",
" return text\n", " return text\n",
"\n", "\n",
"# Retry helper\n", "# Search for posts containing a specific keyword\n",
"def call_with_retry(fn, *args, max_retries=5, base_delay=2, **kwargs):\n", "def get_posts_by_keyword(keyword, since, until, limit=300):\n",
" last_exc = None\n",
" for attempt in range(1, max_retries + 1):\n",
" try:\n", " try:\n",
" return fn(*args, **kwargs)\n", " num_posts = 0\n",
" except Exception as e:\n", " post_data = []\n",
" last_exc = e\n",
" wait = base_delay * (2 ** (attempt - 1)) + random.uniform(0, 1)\n",
" print(f' [retry {attempt}/{max_retries}] {type(e).__name__}: {e} - retrying in {wait:.1f}s')\n",
" time.sleep(wait)\n",
" raise last_exc\n",
"\n", "\n",
"# Post field extraction\n", " progress_bar = tqdm(total=limit, desc=\"Fetching posts\")\n",
"def extract_post(post):\n", "\n",
" try:\n", " while num_posts < limit:\n",
" text = clean_text(post.record.text) if hasattr(post.record, 'text') else None\n",
" iso_date = post.record.created_at if hasattr(post.record, 'created_at') else None\n",
" user = post.author.handle if hasattr(post.author, 'handle') else None\n",
" post_id = post.uri if hasattr(post, 'uri') else None\n",
" parent_id = (\n",
" post.record.reply.parent.uri\n",
" if hasattr(post.record, 'reply') and hasattr(post.record.reply, 'parent')\n",
" else 'no_parent'\n",
" )\n",
" \n",
" if iso_date is None or post_id is None or user is None:\n",
" return None\n",
" \n", " \n",
" search_results = client.app.bsky.feed.search_posts({'q': keyword, 'since': since, 'until': until, 'lang': 'en', 'limit': 100})\n",
" posts = search_results.posts\n",
"\n",
" for post in posts:\n",
" # Extract post details\n",
" text = clean_text(post.record.text) if hasattr(post.record, 'text') else np.nan\n",
" iso_date = post.record.created_at if hasattr(post.record, 'created_at') else np.nan\n",
" user = post.author.handle if hasattr(post.author, 'handle') else np.nan\n",
"\n",
" post_id = post.uri if hasattr(post, 'uri') else np.nan\n",
" parent_id = post.record.reply.parent.uri if hasattr(post.record, 'reply') and hasattr(post.record.reply, 'parent') else 'no_parent'\n",
"\n",
" # Format date-time\n",
" if iso_date != np.nan:\n",
" date_obj = datetime.fromisoformat(iso_date.replace('Z', '+00:00'))\n", " date_obj = datetime.fromisoformat(iso_date.replace('Z', '+00:00'))\n",
" date_str = date_obj.strftime('%Y-%m-%d %H:%M:%S')\n", " date_str = date_obj.strftime('%Y-%m-%d %H:%M:%S')\n",
" \n", " else:\n",
" return {\n", " date_str = np.nan\n",
"\n",
" if post_id != np.nan and user != np.nan:\n",
" post_data.append({\n",
" 'date': date_str,\n", " 'date': date_str,\n",
" 'text': text if text is not None else np.nan,\n", " 'text': text,\n",
" 'user': user,\n", " 'user': user,\n",
" 'id': post_id,\n", " 'id': post_id,\n",
" 'parentid': parent_id,\n", " 'parentid': parent_id\n",
" }, date_obj\n", " })\n",
" except Exception as e:\n", " else:\n",
" print(f\" Skipping malformed post: {e}\")\n", " continue\n",
" return None\n", " \n",
"\n", " until = iso_date\n",
"# Cleanup\n", " num_posts += len(posts)\n",
"def _dedupe(rows):\n", " progress_bar.update(len(posts))\n",
" if not rows:\n", " progress_bar.set_postfix_str(f'now at: {date_str}')\n",
" return pd.DataFrame()\n", "\n",
" df = pd.DataFrame(rows)\n", " if date_obj < (start_timestamp + timedelta(hours=1)):\n",
" before = len(df)\n", " break\n",
" df = df.drop_duplicates(subset='id')\n", "\n",
" df = df.dropna(subset=['text', 'id', 'user'])\n", " progress_bar.close()\n",
" df = df[df['text'].astype(str).str.strip() != '']\n", " df = pd.DataFrame(post_data)\n",
" after = len(df)\n", " return df\n",
" if before != after:\n", "\n",
" print(f\"(Filtered {before - after} duplicate/empty rows out of {before}.)\")\n", " except Exception as e:\n",
" return df.reset_index(drop=True)\n", " print(f\"Error fetching posts: {e}\")\n",
"\n", " return pd.DataFrame()\n",
" \n", "\n",
"# Search for posts containing a specific keyword\n", "# Obtain posts from uri list\n",
"def get_posts_by_keyword(keyword, since, until, limit=300, page_size=100, max_retries=5):\n", "def get_posts(uris):\n",
"\n", " try:\n",
" state = _STATE.setdefault(keyword, {'until': until, 'rows': []})\n", " post_data = []\n",
" \n", "\n",
" if state['until'] != until and state['rows']:\n", " progress_bar = tqdm(total=len(uris), desc=\"Fetching posts\")\n",
" print(f\"Resuming '{keyword}' from until={state['until']}, {len(state['rows'])} posts already fetched this session.\")\n", "\n",
" \n", " for uribit in list(chunked(uris,20)):\n",
" since_dt = datetime.fromisoformat(since.replace('Z', '+00:00'))\n", "\n",
" progress_bar = tqdm(total=limit, initial=min(len(state['rows']), limit), desc=\"Fetching posts\")\n", " search_results = client.app.bsky.feed.get_posts({'uris': uribit})\n",
"\n", " posts = search_results.posts\n",
" try:\n", "\n",
" while len(state['rows']) < limit:\n", " for post in posts: \n",
" try:\n", " # Extract post details\n",
" search_results = call_with_retry(client.app.bsky.feed.search_posts, {'q': keyword, 'since': since, 'until': state['until'], 'lang': 'en', 'limit': page_size}, max_retries=max_retries)\n", " text = clean_text(post.record.text) if hasattr(post.record, 'text') else np.nan\n",
" except Exception as e:\n", " iso_date = post.record.created_at if hasattr(post.record, 'created_at') else np.nan\n",
" print(f\"Giving up on this page after {max_retries} retries: {e}\")\n", " user = post.author.handle if hasattr(post.author, 'handle') else np.nan\n",
" print(\"Progress so far is kept — just call this again to continue.\")\n", "\n",
" break\n", " post_id = post.uri if hasattr(post, 'uri') else np.nan\n",
" \n", " parent_id = post.record.reply.parent.uri if hasattr(post.record, 'reply') and hasattr(post.record.reply, 'parent') else 'no_parent'\n",
" posts = search_results.posts\n", "\n",
" if not posts:\n", " # Format date-time\n",
" print(\"No more posts returned — reached the end of available results.\")\n", " if iso_date != np.nan:\n",
" break\n", " date_obj = datetime.fromisoformat(iso_date.replace('Z', '+00:00'))\n",
" \n", " date_str = date_obj.strftime('%Y-%m-%d %H:%M:%S')\n",
" oldest_date_obj = None\n", " else:\n",
" for post in posts:\n", " date_str = np.nan\n",
" result = extract_post(post)\n", "\n",
" if result is None:\n", " if post_id != np.nan and user != np.nan:\n",
" post_data.append({\n",
" 'date': date_str,\n",
" 'text': text,\n",
" 'user': user,\n",
" 'id': post_id,\n",
" 'parentid': parent_id\n",
" })\n",
" else:\n",
" continue\n", " continue\n",
" row, date_obj = result\n",
" state['rows'].append(row)\n",
" if oldest_date_obj is None or date_obj < oldest_date_obj:\n",
" oldest_date_obj = date_obj\n",
" \n", " \n",
" progress_bar.update(len(posts))\n", " progress_bar.update(len(posts))\n",
" \n", " \n",
" if oldest_date_obj is not None:\n",
" state['until'] = oldest_date_obj.isoformat().replace('+00:00', 'Z')\n",
" progress_bar.set_postfix_str(f'now at: {state[\"until\"]}')\n",
" \n",
" if oldest_date_obj < since_dt + timedelta(hours=1):\n",
" print(\"Reached the `since` boundary.\")\n",
" break\n",
" \n",
" finally:\n",
" progress_bar.close()\n", " progress_bar.close()\n",
" \n", " df = pd.DataFrame(post_data)\n",
" print(f\"{len(state['rows'])} posts fetched in total this session for '{keyword}'. Call again with the same checkpoint_name to keep going.\")\n", " return df\n",
" return _dedupe(state['rows'])\n",
"\n", "\n",
"# Obtain posts from uri list\n",
"def get_posts(uris, chunk_size=20, max_retries=5):\n",
" state = _STATE.setdefault('_by_uri', {'done_uris': set(), 'rows': []})\n",
" \n",
" remaining = [u for u in uris if u not in state['done_uris']]\n",
" chunks = list(chunked(remaining, chunk_size))\n",
" \n",
" progress_bar = tqdm(total=len(uris), initial=len(state['done_uris']), desc=\"Fetching posts\")\n",
" \n",
" try:\n",
" for uribit in chunks:\n",
" try:\n",
" search_results = call_with_retry(client.app.bsky.feed.get_posts, {'uris': uribit}, max_retries=max_retries)\n",
" except Exception as e:\n", " except Exception as e:\n",
" print(f\"Skipping this chunk of {len(uribit)} URIs after {max_retries} retries: {e}\")\n", " print(f\"Error fetching posts: {e}\")\n",
" continue\n", " return pd.DataFrame()"
" \n",
" for post in search_results.posts:\n",
" result = extract_post(post)\n",
" if result is None:\n",
" continue\n",
" row, _ = result\n",
" state['rows'].append(row)\n",
" \n",
" state['done_uris'].update(uribit)\n",
" progress_bar.update(len(uribit))\n",
" \n",
" finally:\n",
" progress_bar.close()\n",
" \n",
" print(f\"{len(state['done_uris'])}/{len(uris)} URIs processed in total.\")\n",
" return _dedupe(state['rows'])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Define state"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"_STATE = {}"
] ]
}, },
{ {
@ -288,7 +233,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"raw_df_data = get_posts_by_keyword('...', iso_start_timestamp, iso_end_timestamp, 100000)\n", "raw_df_data = get_posts_by_keyword('...', iso_start_timestamp, iso_end_timestamp, 500000)\n",
"raw_df_data.info()" "raw_df_data.info()"
] ]
}, },
@ -306,19 +251,13 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"_STATE = {}\n",
"id_list = raw_df_data['id'].to_list()\n", "id_list = raw_df_data['id'].to_list()\n",
"id_list.append('no_parent')\n", "id_list.append('no_parent')\n",
"absent_parents = raw_df_data.loc[~raw_df_data['parentid'].isin(id_list), 'parentid'].tolist()" "\n",
] "absent_parents = raw_df_data.loc[~raw_df_data['parentid'].isin(id_list), 'parentid'].tolist()\n",
}, "\n",
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df_absent_data = get_posts(absent_parents)\n", "df_absent_data = get_posts(absent_parents)\n",
"\n",
"df_absent_data.info()" "df_absent_data.info()"
] ]
}, },
@ -336,7 +275,44 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"df_data = pd.concat([raw_df_data, df_absent_data], ignore_index=True)\n", "df_data = pd.concat(raw_df_data, df_absent_data, ignore_index=True)\n",
"\n",
"df_data.info()"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"Drop duplicates."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df_data = df_data.drop_duplicates()\n",
"df_data.info()"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"Remove empty entries."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df_data.dropna(inplace=True,subset=['text'])\n",
"df_data.info()" "df_data.info()"
] ]
}, },
@ -380,7 +356,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"df_data.to_csv(f'{case}.csv', index=False)" "df_data.to_csv(f'../datasets/{case}.csv', index=False)"
] ]
}, },
{ {
@ -482,7 +458,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"closed_df_reduced_data.to_csv(f'{case}-social-closed.csv', index=False)" "closed_df_reduced_data.to_csv(f'../datasets/{case}-social-closed.csv', index=False)"
] ]
}, },
{ {
@ -499,7 +475,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"closed_df_data.to_csv(f'{case}-closed.csv', index=False)" "closed_df_data.to_csv(f'../datasets/{case}-closed.csv', index=False)"
] ]
} }
], ],

14
graph-construct/construct-graph.ipynb
View file

@ -30,7 +30,7 @@
"case = '...'\n", "case = '...'\n",
"casegraphs = '...'\n", "casegraphs = '...'\n",
"\n", "\n",
"df_data = pd.read_csv(f'../sentiment-analysis/{case}.csv')\n", "df_data = pd.read_csv(f'../datasets/{case}.csv')\n",
"user_nodes = df_data.groupby('user')['id'].apply(list).to_dict()\n", "user_nodes = df_data.groupby('user')['id'].apply(list).to_dict()\n",
"\n", "\n",
"df_data.info()" "df_data.info()"
@ -62,7 +62,7 @@
" G.add_edge(row['id'], row['parentid'])\n", " G.add_edge(row['id'], row['parentid'])\n",
"\n", "\n",
"# Export nearest-neighbour graph\n", "# Export nearest-neighbour graph\n",
"nx.write_graphml(G, f'{casegraphs}-nn-graph.graphml')\n", "nx.write_graphml(G, f'../datasets/{casegraphs}-nn-graph.graphml')\n",
"\n", "\n",
"# Define nearest-neighbour interaction count dictionary\n", "# Define nearest-neighbour interaction count dictionary\n",
"nn_count = {}\n", "nn_count = {}\n",
@ -98,7 +98,7 @@
"tc_G = nx.transitive_closure(G)\n", "tc_G = nx.transitive_closure(G)\n",
"\n", "\n",
"# Export transitive-closure graph\n", "# Export transitive-closure graph\n",
"nx.write_graphml(tc_G, f'{casegraphs}-tc-graph.graphml')\n", "nx.write_graphml(tc_G, f'../datasets/{casegraphs}-tc-graph.graphml')\n",
"\n", "\n",
"# Define transitive-closure interaction count dictionary\n", "# Define transitive-closure interaction count dictionary\n",
"tc_count = {}\n", "tc_count = {}\n",
@ -139,19 +139,19 @@
" if row['parentid'] != 'no_parent':\n", " if row['parentid'] != 'no_parent':\n",
" dG.add_edge(row['id'], row['parentid'])\n", " dG.add_edge(row['id'], row['parentid'])\n",
"\n", "\n",
"nx.write_graphml(dG, f'{casegraphs}-nn-dgraph.graphml')\n", "nx.write_graphml(dG, f'../datasets/{casegraphs}-nn-dgraph.graphml')\n",
"\n", "\n",
"# Construct directed-transitive-closure directed graph\n", "# Construct directed-transitive-closure directed graph\n",
"dtc_dG = nx.transitive_closure(dG)\n", "dtc_dG = nx.transitive_closure(dG)\n",
"\n", "\n",
"# Export directed-transitive-closure directed graph\n", "# Export directed-transitive-closure directed graph\n",
"nx.write_graphml(dtc_dG, f'{casegraphs}-dtc-dgraph.graphml')\n", "nx.write_graphml(dtc_dG, f'../datasets/{casegraphs}-dtc-dgraph.graphml')\n",
"\n", "\n",
"# Convert to directed-transitive-closure undirected graph\n", "# Convert to directed-transitive-closure undirected graph\n",
"dtc_G = dtc_dG.to_undirected()\n", "dtc_G = dtc_dG.to_undirected()\n",
"\n", "\n",
"# Export directed-transitive-closure graph\n", "# Export directed-transitive-closure graph\n",
"nx.write_graphml(dtc_G, f'{casegraphs}-dtc-graph.graphml')\n", "nx.write_graphml(dtc_G, f'../datasets/{casegraphs}-dtc-graph.graphml')\n",
"\n", "\n",
"# Define directed-transitive-closure interaction count dictionary\n", "# Define directed-transitive-closure interaction count dictionary\n",
"dtc_count = {}\n", "dtc_count = {}\n",
@ -196,7 +196,7 @@
" 'post-count': pd.Series(user_c)\n", " 'post-count': pd.Series(user_c)\n",
"})\n", "})\n",
"\n", "\n",
"export_dataset = f'{case}-users.csv'\n", "export_dataset = f'../datasets/{case}-users.csv'\n",
"df_users.to_csv(export_dataset, index=False)\n", "df_users.to_csv(export_dataset, index=False)\n",
"\n", "\n",
"df_users.info()" "df_users.info()"

4
graph-inspect/inspect-graph-comparison.ipynb
View file

@ -33,8 +33,8 @@
"case2 = '...'\n", "case2 = '...'\n",
"\n", "\n",
"# Import the graph\n", "# Import the graph\n",
"G1 = nx.read_graphml(f'../graph-construct/{case1}-dtc-graph.graphml')\n", "G1 = nx.read_graphml(f'../datasets/{case1}-dtc-graph.graphml')\n",
"G2 = nx.read_graphml(f'../graph-construct/{case2}-dtc-graph.graphml')\n", "G2 = nx.read_graphml(f'../datasets/{case2}-dtc-graph.graphml')\n",
"\n", "\n",
"# Print nodes, edges and density of the graphs\n", "# Print nodes, edges and density of the graphs\n",
"print(f'G1:\\nNodes: {G1.number_of_nodes()}\\nEdges: {G1.number_of_edges()}\\nDensity: {nx.density(G1):.5f}')\n", "print(f'G1:\\nNodes: {G1.number_of_nodes()}\\nEdges: {G1.number_of_edges()}\\nDensity: {nx.density(G1):.5f}')\n",

52
sentiment-analysis/sentiment-analysis.py
View file

@ -1,56 +1,30 @@
import os import os
import pandas as pd import pandas as pd
from tqdm import tqdm
import torch import torch
from torch.utils.data import DataLoader, TensorDataset from transformers import DistilBertTokenizerFast
from transformers import ( from torch.utils.data import DataLoader
DistilBertTokenizerFast, from transformers import DistilBertForSequenceClassification
DistilBertForSequenceClassification from tqdm import tqdm
)
# Importing the csv dataset. # Importing the csv dataset.
case = '...' case = '...'
df_data = pd.read_csv(f'{case}.csv') df_data = pd.read_csv(f'../datasets/{case}.csv')
print(len(df_data)) print(len(df_data))
# Set device cuda # Preparing the dataset for the network this includes tokenization, encoding and creating dataloaders.
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('Using:', device)
# Define model
model_name = 'distilbert-base-uncased' model_name = 'distilbert-base-uncased'
tokenizer = DistilBertTokenizerFast.from_pretrained(model_name) tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)
# Tokenize and encode the text data encodings = tokenizer(df_data['text'].tolist(), truncation=True, padding=True, return_tensors='pt')
encodings = tokenizer(
df_data['text'].tolist(),
truncation=True,
padding=True,
return_tensors='pt'
)
# Create tensor dataset dataset = torch.utils.data.TensorDataset(
dataset = TensorDataset(
encodings['input_ids'], encodings['input_ids'],
encodings['attention_mask'] encodings['attention_mask']
) )
dataloader = DataLoader(dataset, batch_size=45, shuffle=False)
# Create data loader # Loading the model.
if device == 'cuda': model= torch.load('models/bert-aclimdb.pth')
en_pin_memory = True
else:
en_pin_memory = False
dataloader = DataLoader(
dataset,
batch_size=53,
shuffle=False,
pin_memory=en_pin_memory
)
# Loading the model
model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=1)
model.load_state_dict(torch.load('...', map_location=device))
model.to(device)
# Using the model to perform sentiment analysis on the dataset. # Using the model to perform sentiment analysis on the dataset.
model.eval() model.eval()
@ -58,7 +32,7 @@ list_predicted_scores = []
for batch in tqdm(dataloader): for batch in tqdm(dataloader):
with torch.no_grad(): with torch.no_grad():
input_ids, attention_mask = [x.to(device) for x in batch] input_ids, attention_mask = batch
# Obtaining the sentiment score. # Obtaining the sentiment score.
output = model(input_ids=input_ids, attention_mask=attention_mask) output = model(input_ids=input_ids, attention_mask=attention_mask)
@ -71,4 +45,4 @@ for batch in tqdm(dataloader):
df_data['s'] = list_predicted_scores df_data['s'] = list_predicted_scores
# Exporting the dataframe to a csv dataset. # Exporting the dataframe to a csv dataset.
df_data.to_csv(f'{case}-s.csv', index=False) df_data.to_csv(f'../datasets/{case}-s.csv', index=False)

82
sentiment-analysis/training/aclimdb-sentiment-training.ipynb
View file

@ -27,8 +27,8 @@
"import os\n", "import os\n",
"import pandas as pd\n", "import pandas as pd\n",
"\n", "\n",
"imdb_train_dataset = 'aclimdb/train'\n", "imdb_train_dataset = \"aclimdb/train\"\n",
"imdb_test_dataset = 'aclimdb/test'\n", "imdb_test_dataset = \"aclimdb/test\"\n",
"\n", "\n",
"train_reviews = []\n", "train_reviews = []\n",
"train_scores = []\n", "train_scores = []\n",
@ -118,7 +118,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"print(f'Length training set: {len(df_train)}\\nLength testing set: {len(df_test)}')" "print(f\"Length training set: {len(df_train)}\\nLength testing set: {len(df_test)}\")"
] ]
}, },
{ {
@ -136,67 +136,31 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"import torch\n", "import torch\n",
"from torch.utils.data import DataLoader, TensorDataset\n", "from transformers import DistilBertTokenizerFast\n",
"from transformers import (\n", "from torch.utils.data import DataLoader\n",
" DistilBertTokenizerFast,\n", "from transformers import DistilBertForSequenceClassification\n",
" DistilBertForSequenceClassification\n",
")\n",
"\n", "\n",
"# Set device cuda\n", "model_name = \"distilbert-base-uncased\"\n",
"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
"print('Using:', device)\n",
"\n",
"# Define model\n",
"model_name = 'distilbert-base-uncased'\n",
"tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)\n", "tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)\n",
"\n", "\n",
"# Tokenize and encode the text data\n", "# Tokenize and encode the text data\n",
"train_encodings = tokenizer(\n", "train_encodings = tokenizer(df_train['text'].tolist(), truncation=True, padding=True, return_tensors='pt')\n",
" df_train['text'].tolist(), \n", "test_encodings = tokenizer(df_test['text'].tolist(), truncation=True, padding=True, return_tensors='pt')\n",
" truncation=True, \n",
" padding=True, \n",
" return_tensors='pt'\n",
")\n",
"\n", "\n",
"test_encodings = tokenizer(\n", "# Create data loaders\n",
" df_test['text'].tolist(), \n", "train_dataset = torch.utils.data.TensorDataset(\n",
" truncation=True, \n",
" padding=True, \n",
" return_tensors='pt'\n",
")\n",
"\n",
"# Create tensor dataset\n",
"train_dataset = TensorDataset(\n",
" train_encodings['input_ids'], \n", " train_encodings['input_ids'], \n",
" train_encodings['attention_mask'], \n", " train_encodings['attention_mask'], \n",
" torch.tensor(df_train['s'].values, dtype=torch.float32)\n", " torch.tensor(df_train['s'].values, dtype=torch.float32)\n",
")\n", ")\n",
"train_dataloader = DataLoader(train_dataset, batch_size=25, shuffle=True)\n",
"\n", "\n",
"test_dataset = TensorDataset(\n", "test_dataset = torch.utils.data.TensorDataset(\n",
" test_encodings['input_ids'], \n", " test_encodings['input_ids'], \n",
" test_encodings['attention_mask'], \n", " test_encodings['attention_mask'], \n",
" torch.tensor(df_test['s'].values, dtype=torch.float32)\n", " torch.tensor(df_test['s'].values, dtype=torch.float32)\n",
")\n", ")\n",
"\n", "test_dataloader = DataLoader(test_dataset, batch_size=25, shuffle=False)"
"# Create data loaders\n",
"if device == 'cuda':\n",
" en_pin_memory = True\n",
"else:\n",
" en_pin_memory = False\n",
"\n",
"train_dataloader = DataLoader(\n",
" train_dataset, \n",
" batch_size=10, \n",
" shuffle=True,\n",
" pin_memory=en_pin_memory\n",
")\n",
"\n",
"test_dataloader = DataLoader(\n",
" test_dataset, \n",
" batch_size=10, \n",
" shuffle=False,\n",
" pin_memory=en_pin_memory\n",
")"
] ]
}, },
{ {
@ -213,10 +177,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = DistilBertForSequenceClassification.from_pretrained(\n", "model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=1)"
" model_name, \n",
" num_labels=1\n",
").to(device)"
] ]
}, },
{ {
@ -274,7 +235,7 @@
" for batch in train_dataloader:\n", " for batch in train_dataloader:\n",
" global_step += 1\n", " global_step += 1\n",
" num_batches += 1\n", " num_batches += 1\n",
" input_ids, attention_mask, target_scores = [x.to(device) for x in batch]\n", " input_ids, attention_mask, target_scores = batch\n",
"\n", "\n",
" # Forward pass\n", " # Forward pass\n",
" output = model(input_ids=input_ids, attention_mask=attention_mask)\n", " output = model(input_ids=input_ids, attention_mask=attention_mask)\n",
@ -309,7 +270,7 @@
"\n", "\n",
" for batch in test_dataloader:\n", " for batch in test_dataloader:\n",
" with torch.no_grad():\n", " with torch.no_grad():\n",
" input_ids, attention_mask, target_scores = [x.to(device) for x in batch]\n", " input_ids, attention_mask, target_scores = batch\n",
"\n", "\n",
" # Obtaining the scores\n", " # Obtaining the scores\n",
" output = model(input_ids=input_ids, attention_mask=attention_mask)\n", " output = model(input_ids=input_ids, attention_mask=attention_mask)\n",
@ -329,7 +290,7 @@
" print(f\"Epoch {epoch + 1}/{num_epochs}, Validation loss: {average_loss:.4f}\\n\")\n", " print(f\"Epoch {epoch + 1}/{num_epochs}, Validation loss: {average_loss:.4f}\\n\")\n",
"\n", "\n",
" # Saving the model\n", " # Saving the model\n",
" torch.save(model.state_dict(), f'bert-aclimdb/{epoch + 1}.pth')\n", " torch.save(model, f'bert-aclimdb/{epoch + 1}.pth')\n",
"\n", "\n",
" # Early stopping check\n", " # Early stopping check\n",
" if average_loss < best_validation_loss:\n", " if average_loss < best_validation_loss:\n",
@ -339,7 +300,6 @@
" no_improvement_counter += 1\n", " no_improvement_counter += 1\n",
"\n", "\n",
" if no_improvement_counter >= early_stop_patience:\n", " if no_improvement_counter >= early_stop_patience:\n",
" print('Early stopping triggered')\n",
" break\n", " break\n",
"\n", "\n",
"writer.close()" "writer.close()"
@ -359,9 +319,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=1)\n", "model = torch.load('bert-aclimdb/4.pth')"
"model.load_state_dict(torch.load('...', map_location=device))\n",
"model.to(device)"
] ]
}, },
{ {
@ -386,7 +344,7 @@
"\n", "\n",
"for batch in test_dataloader:\n", "for batch in test_dataloader:\n",
" with torch.no_grad():\n", " with torch.no_grad():\n",
" input_ids, attention_mask, scores = [x.to(device) for x in batch]\n", " input_ids, attention_mask, scores = batch\n",
"\n", "\n",
" # Obtaining the scores\n", " # Obtaining the scores\n",
" output = model(input_ids=input_ids, attention_mask=attention_mask)\n", " output = model(input_ids=input_ids, attention_mask=attention_mask)\n",