physics/social-graphs
1
0
Fork 0
Code Issues Pull requests Activity

Compare commits

base: physics:8ac5aa2cb3bce0d58ecb724cea0fa0b3157ba52c
Branches Tags
physics:main
..
compare: physics:3c7e3fc018df2b74648ea40592d16540a62d2df4
Branches Tags
physics:main

No commits in common. "8ac5aa2cb3bce0d58ecb724cea0fa0b3157ba52c" and "3c7e3fc018df2b74648ea40592d16540a62d2df4" have entirely different histories.
8ac5aa2cb3 ... 3c7e3fc018

5 changed files with 182 additions and 274 deletions
Download patch file Download diff file Expand all files Collapse all files

304
dataset-retrieval/retrieving-bluesky-datasets.ipynb
View file

@ -98,16 +98,14 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"import re, random, time\n",
"import pandas as pd\n", "import pandas as pd\n",
"import numpy as np\n", "import numpy as np\n",
"import re\n",
"from tqdm import tqdm\n", "from tqdm import tqdm\n",
"from more_itertools import chunked\n", "from more_itertools import chunked\n",
"\n", "\n",
"# Remove URLs, hashtags, mentions, emojis and whitespaces\n", "# Remove URLs, hashtags, mentions, emojis and whitespaces\n",
"def clean_text(text):\n", "def clean_text(text):\n",
" if not isinstance(text, str):\n",
" return np.nan\n",
" text = re.sub(r'http\\S+|www\\S+|https\\S+', '', text, flags=re.MULTILINE)\n", " text = re.sub(r'http\\S+|www\\S+|https\\S+', '', text, flags=re.MULTILINE)\n",
" text = re.sub(r'#\\S+', '', text)\n", " text = re.sub(r'#\\S+', '', text)\n",
" text = re.sub(r'@\\S+', '', text)\n", " text = re.sub(r'@\\S+', '', text)\n",
@ -115,163 +113,110 @@
" text = re.sub(r'\\s+', ' ', text).strip()\n", " text = re.sub(r'\\s+', ' ', text).strip()\n",
" return text\n", " return text\n",
"\n", "\n",
"# Retry helper\n",
"def call_with_retry(fn, *args, max_retries=5, base_delay=2, **kwargs):\n",
" last_exc = None\n",
" for attempt in range(1, max_retries + 1):\n",
" try:\n",
" return fn(*args, **kwargs)\n",
" except Exception as e:\n",
" last_exc = e\n",
" wait = base_delay * (2 ** (attempt - 1)) + random.uniform(0, 1)\n",
" print(f' [retry {attempt}/{max_retries}] {type(e).__name__}: {e} - retrying in {wait:.1f}s')\n",
" time.sleep(wait)\n",
" raise last_exc\n",
"\n",
"# Post field extraction\n",
"def extract_post(post):\n",
" try:\n",
" text = clean_text(post.record.text) if hasattr(post.record, 'text') else None\n",
" iso_date = post.record.created_at if hasattr(post.record, 'created_at') else None\n",
" user = post.author.handle if hasattr(post.author, 'handle') else None\n",
" post_id = post.uri if hasattr(post, 'uri') else None\n",
" parent_id = (\n",
" post.record.reply.parent.uri\n",
" if hasattr(post.record, 'reply') and hasattr(post.record.reply, 'parent')\n",
" else 'no_parent'\n",
" )\n",
" \n",
" if iso_date is None or post_id is None or user is None:\n",
" return None\n",
" \n",
" date_obj = datetime.fromisoformat(iso_date.replace('Z', '+00:00'))\n",
" date_str = date_obj.strftime('%Y-%m-%d %H:%M:%S')\n",
" \n",
" return {\n",
" 'date': date_str,\n",
" 'text': text if text is not None else np.nan,\n",
" 'user': user,\n",
" 'id': post_id,\n",
" 'parentid': parent_id,\n",
" }, date_obj\n",
" except Exception as e:\n",
" print(f\" Skipping malformed post: {e}\")\n",
" return None\n",
"\n",
"# Cleanup\n",
"def _dedupe(rows):\n",
" if not rows:\n",
" return pd.DataFrame()\n",
" df = pd.DataFrame(rows)\n",
" before = len(df)\n",
" df = df.drop_duplicates(subset='id')\n",
" df = df.dropna(subset=['text', 'id', 'user'])\n",
" df = df[df['text'].astype(str).str.strip() != '']\n",
" after = len(df)\n",
" if before != after:\n",
" print(f\"(Filtered {before - after} duplicate/empty rows out of {before}.)\")\n",
" return df.reset_index(drop=True)\n",
"\n",
" \n",
"# Search for posts containing a specific keyword\n", "# Search for posts containing a specific keyword\n",
"def get_posts_by_keyword(keyword, since, until, limit=300, page_size=100, max_retries=5):\n", "def get_posts_by_keyword(keyword, since, until, limit=300):\n",
"\n",
" state = _STATE.setdefault(keyword, {'until': until, 'rows': []})\n",
" \n",
" if state['until'] != until and state['rows']:\n",
" print(f\"Resuming '{keyword}' from until={state['until']}, {len(state['rows'])} posts already fetched this session.\")\n",
" \n",
" since_dt = datetime.fromisoformat(since.replace('Z', '+00:00'))\n",
" progress_bar = tqdm(total=limit, initial=min(len(state['rows']), limit), desc=\"Fetching posts\")\n",
"\n",
" try:\n", " try:\n",
" while len(state['rows']) < limit:\n", " num_posts = 0\n",
" try:\n", " post_data = []\n",
" search_results = call_with_retry(client.app.bsky.feed.search_posts, {'q': keyword, 'since': since, 'until': state['until'], 'lang': 'en', 'limit': page_size}, max_retries=max_retries)\n", "\n",
" except Exception as e:\n", " progress_bar = tqdm(total=limit, desc=\"Fetching posts\")\n",
" print(f\"Giving up on this page after {max_retries} retries: {e}\")\n", "\n",
" print(\"Progress so far is kept — just call this again to continue.\")\n", " while num_posts < limit:\n",
" break\n", " \n",
" \n", " search_results = client.app.bsky.feed.search_posts({'q': keyword, 'since': since, 'until': until, 'lang': 'en', 'limit': 100})\n",
" posts = search_results.posts\n", " posts = search_results.posts\n",
" if not posts:\n", "\n",
" print(\"No more posts returned — reached the end of available results.\")\n",
" break\n",
" \n",
" oldest_date_obj = None\n",
" for post in posts:\n", " for post in posts:\n",
" result = extract_post(post)\n", " # Extract post details\n",
" if result is None:\n", " text = clean_text(post.record.text) if hasattr(post.record, 'text') else np.nan\n",
" continue\n", " iso_date = post.record.created_at if hasattr(post.record, 'created_at') else np.nan\n",
" row, date_obj = result\n", " user = post.author.handle if hasattr(post.author, 'handle') else np.nan\n",
" state['rows'].append(row)\n", "\n",
" if oldest_date_obj is None or date_obj < oldest_date_obj:\n", " post_id = post.uri if hasattr(post, 'uri') else np.nan\n",
" oldest_date_obj = date_obj\n", " parent_id = post.record.reply.parent.uri if hasattr(post.record, 'reply') and hasattr(post.record.reply, 'parent') else 'no_parent'\n",
" \n", "\n",
" # Format date-time\n",
" if iso_date != np.nan:\n",
" date_obj = datetime.fromisoformat(iso_date.replace('Z', '+00:00'))\n",
" date_str = date_obj.strftime('%Y-%m-%d %H:%M:%S')\n",
" else:\n",
" date_str = np.nan\n",
"\n",
" if post_id != np.nan and user != np.nan:\n",
" post_data.append({\n",
" 'date': date_str,\n",
" 'text': text,\n",
" 'user': user,\n",
" 'id': post_id,\n",
" 'parentid': parent_id\n",
" })\n",
" else:\n",
" continue\n",
" \n",
" until = iso_date\n",
" num_posts += len(posts)\n",
" progress_bar.update(len(posts))\n", " progress_bar.update(len(posts))\n",
" \n", " progress_bar.set_postfix_str(f'now at: {date_str}')\n",
" if oldest_date_obj is not None:\n", "\n",
" state['until'] = oldest_date_obj.isoformat().replace('+00:00', 'Z')\n", " if date_obj < (start_timestamp + timedelta(hours=1)):\n",
" progress_bar.set_postfix_str(f'now at: {state[\"until\"]}')\n", " break\n",
" \n", "\n",
" if oldest_date_obj < since_dt + timedelta(hours=1):\n",
" print(\"Reached the `since` boundary.\")\n",
" break\n",
" \n",
" finally:\n",
" progress_bar.close()\n", " progress_bar.close()\n",
" \n", " df = pd.DataFrame(post_data)\n",
" print(f\"{len(state['rows'])} posts fetched in total this session for '{keyword}'. Call again with the same checkpoint_name to keep going.\")\n", " return df\n",
" return _dedupe(state['rows'])\n", "\n",
" except Exception as e:\n",
" print(f\"Error fetching posts: {e}\")\n",
" return pd.DataFrame()\n",
"\n", "\n",
"# Obtain posts from uri list\n", "# Obtain posts from uri list\n",
"def get_posts(uris, chunk_size=20, max_retries=5):\n", "def get_posts(uris):\n",
" state = _STATE.setdefault('_by_uri', {'done_uris': set(), 'rows': []})\n",
" \n",
" remaining = [u for u in uris if u not in state['done_uris']]\n",
" chunks = list(chunked(remaining, chunk_size))\n",
" \n",
" progress_bar = tqdm(total=len(uris), initial=len(state['done_uris']), desc=\"Fetching posts\")\n",
" \n",
" try:\n", " try:\n",
" for uribit in chunks:\n", " post_data = []\n",
" try:\n", "\n",
" search_results = call_with_retry(client.app.bsky.feed.get_posts, {'uris': uribit}, max_retries=max_retries)\n", " progress_bar = tqdm(total=len(uris), desc=\"Fetching posts\")\n",
" except Exception as e:\n", "\n",
" print(f\"Skipping this chunk of {len(uribit)} URIs after {max_retries} retries: {e}\")\n", " for uribit in list(chunked(uris,20)):\n",
" continue\n", "\n",
" \n", " search_results = client.app.bsky.feed.get_posts({'uris': uribit})\n",
" for post in search_results.posts:\n", " posts = search_results.posts\n",
" result = extract_post(post)\n", "\n",
" if result is None:\n", " for post in posts: \n",
" # Extract post details\n",
" text = clean_text(post.record.text) if hasattr(post.record, 'text') else np.nan\n",
" iso_date = post.record.created_at if hasattr(post.record, 'created_at') else np.nan\n",
" user = post.author.handle if hasattr(post.author, 'handle') else np.nan\n",
"\n",
" post_id = post.uri if hasattr(post, 'uri') else np.nan\n",
" parent_id = post.record.reply.parent.uri if hasattr(post.record, 'reply') and hasattr(post.record.reply, 'parent') else 'no_parent'\n",
"\n",
" # Format date-time\n",
" if iso_date != np.nan:\n",
" date_obj = datetime.fromisoformat(iso_date.replace('Z', '+00:00'))\n",
" date_str = date_obj.strftime('%Y-%m-%d %H:%M:%S')\n",
" else:\n",
" date_str = np.nan\n",
"\n",
" if post_id != np.nan and user != np.nan:\n",
" post_data.append({\n",
" 'date': date_str,\n",
" 'text': text,\n",
" 'user': user,\n",
" 'id': post_id,\n",
" 'parentid': parent_id\n",
" })\n",
" else:\n",
" continue\n", " continue\n",
" row, _ = result\n", " \n",
" state['rows'].append(row)\n", " progress_bar.update(len(posts))\n",
" \n", " \n",
" state['done_uris'].update(uribit)\n",
" progress_bar.update(len(uribit))\n",
" \n",
" finally:\n",
" progress_bar.close()\n", " progress_bar.close()\n",
" \n", " df = pd.DataFrame(post_data)\n",
" print(f\"{len(state['done_uris'])}/{len(uris)} URIs processed in total.\")\n", " return df\n",
" return _dedupe(state['rows'])" "\n",
] " except Exception as e:\n",
}, " print(f\"Error fetching posts: {e}\")\n",
{ " return pd.DataFrame()"
"cell_type": "markdown",
"metadata": {},
"source": [
"Define state"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"_STATE = {}"
] ]
}, },
{ {
@ -288,7 +233,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"raw_df_data = get_posts_by_keyword('...', iso_start_timestamp, iso_end_timestamp, 100000)\n", "raw_df_data = get_posts_by_keyword('...', iso_start_timestamp, iso_end_timestamp, 500000)\n",
"raw_df_data.info()" "raw_df_data.info()"
] ]
}, },
@ -306,19 +251,13 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"_STATE = {}\n",
"id_list = raw_df_data['id'].to_list()\n", "id_list = raw_df_data['id'].to_list()\n",
"id_list.append('no_parent')\n", "id_list.append('no_parent')\n",
"absent_parents = raw_df_data.loc[~raw_df_data['parentid'].isin(id_list), 'parentid'].tolist()" "\n",
] "absent_parents = raw_df_data.loc[~raw_df_data['parentid'].isin(id_list), 'parentid'].tolist()\n",
}, "\n",
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df_absent_data = get_posts(absent_parents)\n", "df_absent_data = get_posts(absent_parents)\n",
"\n",
"df_absent_data.info()" "df_absent_data.info()"
] ]
}, },
@ -336,7 +275,44 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"df_data = pd.concat([raw_df_data, df_absent_data], ignore_index=True)\n", "df_data = pd.concat(raw_df_data, df_absent_data, ignore_index=True)\n",
"\n",
"df_data.info()"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"Drop duplicates."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df_data = df_data.drop_duplicates()\n",
"df_data.info()"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"Remove empty entries."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df_data.dropna(inplace=True,subset=['text'])\n",
"df_data.info()" "df_data.info()"
] ]
}, },
@ -380,7 +356,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"df_data.to_csv(f'{case}.csv', index=False)" "df_data.to_csv(f'../datasets/{case}.csv', index=False)"
] ]
}, },
{ {
@ -482,7 +458,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"closed_df_reduced_data.to_csv(f'{case}-social-closed.csv', index=False)" "closed_df_reduced_data.to_csv(f'../datasets/{case}-social-closed.csv', index=False)"
] ]
}, },
{ {
@ -499,7 +475,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"closed_df_data.to_csv(f'{case}-closed.csv', index=False)" "closed_df_data.to_csv(f'../datasets/{case}-closed.csv', index=False)"
] ]
} }
], ],

14
graph-construct/construct-graph.ipynb
View file

@ -30,7 +30,7 @@
"case = '...'\n", "case = '...'\n",
"casegraphs = '...'\n", "casegraphs = '...'\n",
"\n", "\n",
"df_data = pd.read_csv(f'../sentiment-analysis/{case}.csv')\n", "df_data = pd.read_csv(f'../datasets/{case}.csv')\n",
"user_nodes = df_data.groupby('user')['id'].apply(list).to_dict()\n", "user_nodes = df_data.groupby('user')['id'].apply(list).to_dict()\n",
"\n", "\n",
"df_data.info()" "df_data.info()"
@ -62,7 +62,7 @@
" G.add_edge(row['id'], row['parentid'])\n", " G.add_edge(row['id'], row['parentid'])\n",
"\n", "\n",
"# Export nearest-neighbour graph\n", "# Export nearest-neighbour graph\n",
"nx.write_graphml(G, f'{casegraphs}-nn-graph.graphml')\n", "nx.write_graphml(G, f'../datasets/{casegraphs}-nn-graph.graphml')\n",
"\n", "\n",
"# Define nearest-neighbour interaction count dictionary\n", "# Define nearest-neighbour interaction count dictionary\n",
"nn_count = {}\n", "nn_count = {}\n",
@ -98,7 +98,7 @@
"tc_G = nx.transitive_closure(G)\n", "tc_G = nx.transitive_closure(G)\n",
"\n", "\n",
"# Export transitive-closure graph\n", "# Export transitive-closure graph\n",
"nx.write_graphml(tc_G, f'{casegraphs}-tc-graph.graphml')\n", "nx.write_graphml(tc_G, f'../datasets/{casegraphs}-tc-graph.graphml')\n",
"\n", "\n",
"# Define transitive-closure interaction count dictionary\n", "# Define transitive-closure interaction count dictionary\n",
"tc_count = {}\n", "tc_count = {}\n",
@ -139,19 +139,19 @@
" if row['parentid'] != 'no_parent':\n", " if row['parentid'] != 'no_parent':\n",
" dG.add_edge(row['id'], row['parentid'])\n", " dG.add_edge(row['id'], row['parentid'])\n",
"\n", "\n",
"nx.write_graphml(dG, f'{casegraphs}-nn-dgraph.graphml')\n", "nx.write_graphml(dG, f'../datasets/{casegraphs}-nn-dgraph.graphml')\n",
"\n", "\n",
"# Construct directed-transitive-closure directed graph\n", "# Construct directed-transitive-closure directed graph\n",
"dtc_dG = nx.transitive_closure(dG)\n", "dtc_dG = nx.transitive_closure(dG)\n",
"\n", "\n",
"# Export directed-transitive-closure directed graph\n", "# Export directed-transitive-closure directed graph\n",
"nx.write_graphml(dtc_dG, f'{casegraphs}-dtc-dgraph.graphml')\n", "nx.write_graphml(dtc_dG, f'../datasets/{casegraphs}-dtc-dgraph.graphml')\n",
"\n", "\n",
"# Convert to directed-transitive-closure undirected graph\n", "# Convert to directed-transitive-closure undirected graph\n",
"dtc_G = dtc_dG.to_undirected()\n", "dtc_G = dtc_dG.to_undirected()\n",
"\n", "\n",
"# Export directed-transitive-closure graph\n", "# Export directed-transitive-closure graph\n",
"nx.write_graphml(dtc_G, f'{casegraphs}-dtc-graph.graphml')\n", "nx.write_graphml(dtc_G, f'../datasets/{casegraphs}-dtc-graph.graphml')\n",
"\n", "\n",
"# Define directed-transitive-closure interaction count dictionary\n", "# Define directed-transitive-closure interaction count dictionary\n",
"dtc_count = {}\n", "dtc_count = {}\n",
@ -196,7 +196,7 @@
" 'post-count': pd.Series(user_c)\n", " 'post-count': pd.Series(user_c)\n",
"})\n", "})\n",
"\n", "\n",
"export_dataset = f'{case}-users.csv'\n", "export_dataset = f'../datasets/{case}-users.csv'\n",
"df_users.to_csv(export_dataset, index=False)\n", "df_users.to_csv(export_dataset, index=False)\n",
"\n", "\n",
"df_users.info()" "df_users.info()"

4
graph-inspect/inspect-graph-comparison.ipynb
View file

@ -33,8 +33,8 @@
"case2 = '...'\n", "case2 = '...'\n",
"\n", "\n",
"# Import the graph\n", "# Import the graph\n",
"G1 = nx.read_graphml(f'../graph-construct/{case1}-dtc-graph.graphml')\n", "G1 = nx.read_graphml(f'../datasets/{case1}-dtc-graph.graphml')\n",
"G2 = nx.read_graphml(f'../graph-construct/{case2}-dtc-graph.graphml')\n", "G2 = nx.read_graphml(f'../datasets/{case2}-dtc-graph.graphml')\n",
"\n", "\n",
"# Print nodes, edges and density of the graphs\n", "# Print nodes, edges and density of the graphs\n",
"print(f'G1:\\nNodes: {G1.number_of_nodes()}\\nEdges: {G1.number_of_edges()}\\nDensity: {nx.density(G1):.5f}')\n", "print(f'G1:\\nNodes: {G1.number_of_nodes()}\\nEdges: {G1.number_of_edges()}\\nDensity: {nx.density(G1):.5f}')\n",

52
sentiment-analysis/sentiment-analysis.py
View file

@ -1,56 +1,30 @@
import os import os
import pandas as pd import pandas as pd
from tqdm import tqdm
import torch import torch
from torch.utils.data import DataLoader, TensorDataset from transformers import DistilBertTokenizerFast
from transformers import ( from torch.utils.data import DataLoader
DistilBertTokenizerFast, from transformers import DistilBertForSequenceClassification
DistilBertForSequenceClassification from tqdm import tqdm
)
# Importing the csv dataset. # Importing the csv dataset.
case = '...' case = '...'
df_data = pd.read_csv(f'{case}.csv') df_data = pd.read_csv(f'../datasets/{case}.csv')
print(len(df_data)) print(len(df_data))
# Set device cuda # Preparing the dataset for the network this includes tokenization, encoding and creating dataloaders.
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('Using:', device)
# Define model
model_name = 'distilbert-base-uncased' model_name = 'distilbert-base-uncased'
tokenizer = DistilBertTokenizerFast.from_pretrained(model_name) tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)
# Tokenize and encode the text data encodings = tokenizer(df_data['text'].tolist(), truncation=True, padding=True, return_tensors='pt')
encodings = tokenizer(
df_data['text'].tolist(),
truncation=True,
padding=True,
return_tensors='pt'
)
# Create tensor dataset dataset = torch.utils.data.TensorDataset(
dataset = TensorDataset(
encodings['input_ids'], encodings['input_ids'],
encodings['attention_mask'] encodings['attention_mask']
) )
dataloader = DataLoader(dataset, batch_size=45, shuffle=False)
# Create data loader # Loading the model.
if device == 'cuda': model= torch.load('models/bert-aclimdb.pth')
en_pin_memory = True
else:
en_pin_memory = False
dataloader = DataLoader(
dataset,
batch_size=53,
shuffle=False,
pin_memory=en_pin_memory
)
# Loading the model
model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=1)
model.load_state_dict(torch.load('...', map_location=device))
model.to(device)
# Using the model to perform sentiment analysis on the dataset. # Using the model to perform sentiment analysis on the dataset.
model.eval() model.eval()
@ -58,7 +32,7 @@ list_predicted_scores = []
for batch in tqdm(dataloader): for batch in tqdm(dataloader):
with torch.no_grad(): with torch.no_grad():
input_ids, attention_mask = [x.to(device) for x in batch] input_ids, attention_mask = batch
# Obtaining the sentiment score. # Obtaining the sentiment score.
output = model(input_ids=input_ids, attention_mask=attention_mask) output = model(input_ids=input_ids, attention_mask=attention_mask)
@ -71,4 +45,4 @@ for batch in tqdm(dataloader):
df_data['s'] = list_predicted_scores df_data['s'] = list_predicted_scores
# Exporting the dataframe to a csv dataset. # Exporting the dataframe to a csv dataset.
df_data.to_csv(f'{case}-s.csv', index=False) df_data.to_csv(f'../datasets/{case}-s.csv', index=False)

82
sentiment-analysis/training/aclimdb-sentiment-training.ipynb
View file

@ -27,8 +27,8 @@
"import os\n", "import os\n",
"import pandas as pd\n", "import pandas as pd\n",
"\n", "\n",
"imdb_train_dataset = 'aclimdb/train'\n", "imdb_train_dataset = \"aclimdb/train\"\n",
"imdb_test_dataset = 'aclimdb/test'\n", "imdb_test_dataset = \"aclimdb/test\"\n",
"\n", "\n",
"train_reviews = []\n", "train_reviews = []\n",
"train_scores = []\n", "train_scores = []\n",
@ -118,7 +118,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"print(f'Length training set: {len(df_train)}\\nLength testing set: {len(df_test)}')" "print(f\"Length training set: {len(df_train)}\\nLength testing set: {len(df_test)}\")"
] ]
}, },
{ {
@ -136,67 +136,31 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"import torch\n", "import torch\n",
"from torch.utils.data import DataLoader, TensorDataset\n", "from transformers import DistilBertTokenizerFast\n",
"from transformers import (\n", "from torch.utils.data import DataLoader\n",
" DistilBertTokenizerFast,\n", "from transformers import DistilBertForSequenceClassification\n",
" DistilBertForSequenceClassification\n",
")\n",
"\n", "\n",
"# Set device cuda\n", "model_name = \"distilbert-base-uncased\"\n",
"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
"print('Using:', device)\n",
"\n",
"# Define model\n",
"model_name = 'distilbert-base-uncased'\n",
"tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)\n", "tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)\n",
"\n", "\n",
"# Tokenize and encode the text data\n", "# Tokenize and encode the text data\n",
"train_encodings = tokenizer(\n", "train_encodings = tokenizer(df_train['text'].tolist(), truncation=True, padding=True, return_tensors='pt')\n",
" df_train['text'].tolist(), \n", "test_encodings = tokenizer(df_test['text'].tolist(), truncation=True, padding=True, return_tensors='pt')\n",
" truncation=True, \n",
" padding=True, \n",
" return_tensors='pt'\n",
")\n",
"\n", "\n",
"test_encodings = tokenizer(\n", "# Create data loaders\n",
" df_test['text'].tolist(), \n", "train_dataset = torch.utils.data.TensorDataset(\n",
" truncation=True, \n",
" padding=True, \n",
" return_tensors='pt'\n",
")\n",
"\n",
"# Create tensor dataset\n",
"train_dataset = TensorDataset(\n",
" train_encodings['input_ids'], \n", " train_encodings['input_ids'], \n",
" train_encodings['attention_mask'], \n", " train_encodings['attention_mask'], \n",
" torch.tensor(df_train['s'].values, dtype=torch.float32)\n", " torch.tensor(df_train['s'].values, dtype=torch.float32)\n",
")\n", ")\n",
"train_dataloader = DataLoader(train_dataset, batch_size=25, shuffle=True)\n",
"\n", "\n",
"test_dataset = TensorDataset(\n", "test_dataset = torch.utils.data.TensorDataset(\n",
" test_encodings['input_ids'], \n", " test_encodings['input_ids'], \n",
" test_encodings['attention_mask'], \n", " test_encodings['attention_mask'], \n",
" torch.tensor(df_test['s'].values, dtype=torch.float32)\n", " torch.tensor(df_test['s'].values, dtype=torch.float32)\n",
")\n", ")\n",
"\n", "test_dataloader = DataLoader(test_dataset, batch_size=25, shuffle=False)"
"# Create data loaders\n",
"if device == 'cuda':\n",
" en_pin_memory = True\n",
"else:\n",
" en_pin_memory = False\n",
"\n",
"train_dataloader = DataLoader(\n",
" train_dataset, \n",
" batch_size=10, \n",
" shuffle=True,\n",
" pin_memory=en_pin_memory\n",
")\n",
"\n",
"test_dataloader = DataLoader(\n",
" test_dataset, \n",
" batch_size=10, \n",
" shuffle=False,\n",
" pin_memory=en_pin_memory\n",
")"
] ]
}, },
{ {
@ -213,10 +177,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = DistilBertForSequenceClassification.from_pretrained(\n", "model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=1)"
" model_name, \n",
" num_labels=1\n",
").to(device)"
] ]
}, },
{ {
@ -274,7 +235,7 @@
" for batch in train_dataloader:\n", " for batch in train_dataloader:\n",
" global_step += 1\n", " global_step += 1\n",
" num_batches += 1\n", " num_batches += 1\n",
" input_ids, attention_mask, target_scores = [x.to(device) for x in batch]\n", " input_ids, attention_mask, target_scores = batch\n",
"\n", "\n",
" # Forward pass\n", " # Forward pass\n",
" output = model(input_ids=input_ids, attention_mask=attention_mask)\n", " output = model(input_ids=input_ids, attention_mask=attention_mask)\n",
@ -309,7 +270,7 @@
"\n", "\n",
" for batch in test_dataloader:\n", " for batch in test_dataloader:\n",
" with torch.no_grad():\n", " with torch.no_grad():\n",
" input_ids, attention_mask, target_scores = [x.to(device) for x in batch]\n", " input_ids, attention_mask, target_scores = batch\n",
"\n", "\n",
" # Obtaining the scores\n", " # Obtaining the scores\n",
" output = model(input_ids=input_ids, attention_mask=attention_mask)\n", " output = model(input_ids=input_ids, attention_mask=attention_mask)\n",
@ -329,7 +290,7 @@
" print(f\"Epoch {epoch + 1}/{num_epochs}, Validation loss: {average_loss:.4f}\\n\")\n", " print(f\"Epoch {epoch + 1}/{num_epochs}, Validation loss: {average_loss:.4f}\\n\")\n",
"\n", "\n",
" # Saving the model\n", " # Saving the model\n",
" torch.save(model.state_dict(), f'bert-aclimdb/{epoch + 1}.pth')\n", " torch.save(model, f'bert-aclimdb/{epoch + 1}.pth')\n",
"\n", "\n",
" # Early stopping check\n", " # Early stopping check\n",
" if average_loss < best_validation_loss:\n", " if average_loss < best_validation_loss:\n",
@ -339,7 +300,6 @@
" no_improvement_counter += 1\n", " no_improvement_counter += 1\n",
"\n", "\n",
" if no_improvement_counter >= early_stop_patience:\n", " if no_improvement_counter >= early_stop_patience:\n",
" print('Early stopping triggered')\n",
" break\n", " break\n",
"\n", "\n",
"writer.close()" "writer.close()"
@ -359,9 +319,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=1)\n", "model = torch.load('bert-aclimdb/4.pth')"
"model.load_state_dict(torch.load('...', map_location=device))\n",
"model.to(device)"
] ]
}, },
{ {
@ -386,7 +344,7 @@
"\n", "\n",
"for batch in test_dataloader:\n", "for batch in test_dataloader:\n",
" with torch.no_grad():\n", " with torch.no_grad():\n",
" input_ids, attention_mask, scores = [x.to(device) for x in batch]\n", " input_ids, attention_mask, scores = batch\n",
"\n", "\n",
" # Obtaining the scores\n", " # Obtaining the scores\n",
" output = model(input_ids=input_ids, attention_mask=attention_mask)\n", " output = model(input_ids=input_ids, attention_mask=attention_mask)\n",