diff --git a/dataset-retrieval/retrieving-bluesky-datasets.ipynb b/dataset-retrieval/retrieving-bluesky-datasets.ipynb index 81d8918..9a31618 100644 --- a/dataset-retrieval/retrieving-bluesky-datasets.ipynb +++ b/dataset-retrieval/retrieving-bluesky-datasets.ipynb @@ -98,16 +98,14 @@ "metadata": {}, "outputs": [], "source": [ - "import re, random, time\n", "import pandas as pd\n", "import numpy as np\n", + "import re\n", "from tqdm import tqdm\n", "from more_itertools import chunked\n", "\n", "# Remove URLs, hashtags, mentions, emojis and whitespaces\n", "def clean_text(text):\n", - " if not isinstance(text, str):\n", - " return np.nan\n", " text = re.sub(r'http\\S+|www\\S+|https\\S+', '', text, flags=re.MULTILINE)\n", " text = re.sub(r'#\\S+', '', text)\n", " text = re.sub(r'@\\S+', '', text)\n", @@ -115,163 +113,110 @@ " text = re.sub(r'\\s+', ' ', text).strip()\n", " return text\n", "\n", - "# Retry helper\n", - "def call_with_retry(fn, *args, max_retries=5, base_delay=2, **kwargs):\n", - " last_exc = None\n", - " for attempt in range(1, max_retries + 1):\n", - " try:\n", - " return fn(*args, **kwargs)\n", - " except Exception as e:\n", - " last_exc = e\n", - " wait = base_delay * (2 ** (attempt - 1)) + random.uniform(0, 1)\n", - " print(f' [retry {attempt}/{max_retries}] {type(e).__name__}: {e} - retrying in {wait:.1f}s')\n", - " time.sleep(wait)\n", - " raise last_exc\n", - "\n", - "# Post field extraction\n", - "def extract_post(post):\n", - " try:\n", - " text = clean_text(post.record.text) if hasattr(post.record, 'text') else None\n", - " iso_date = post.record.created_at if hasattr(post.record, 'created_at') else None\n", - " user = post.author.handle if hasattr(post.author, 'handle') else None\n", - " post_id = post.uri if hasattr(post, 'uri') else None\n", - " parent_id = (\n", - " post.record.reply.parent.uri\n", - " if hasattr(post.record, 'reply') and hasattr(post.record.reply, 'parent')\n", - " else 'no_parent'\n", - " )\n", - " \n", - " if iso_date is None or post_id is None or user is None:\n", - " return None\n", - " \n", - " date_obj = datetime.fromisoformat(iso_date.replace('Z', '+00:00'))\n", - " date_str = date_obj.strftime('%Y-%m-%d %H:%M:%S')\n", - " \n", - " return {\n", - " 'date': date_str,\n", - " 'text': text if text is not None else np.nan,\n", - " 'user': user,\n", - " 'id': post_id,\n", - " 'parentid': parent_id,\n", - " }, date_obj\n", - " except Exception as e:\n", - " print(f\" Skipping malformed post: {e}\")\n", - " return None\n", - "\n", - "# Cleanup\n", - "def _dedupe(rows):\n", - " if not rows:\n", - " return pd.DataFrame()\n", - " df = pd.DataFrame(rows)\n", - " before = len(df)\n", - " df = df.drop_duplicates(subset='id')\n", - " df = df.dropna(subset=['text', 'id', 'user'])\n", - " df = df[df['text'].astype(str).str.strip() != '']\n", - " after = len(df)\n", - " if before != after:\n", - " print(f\"(Filtered {before - after} duplicate/empty rows out of {before}.)\")\n", - " return df.reset_index(drop=True)\n", - "\n", - " \n", "# Search for posts containing a specific keyword\n", - "def get_posts_by_keyword(keyword, since, until, limit=300, page_size=100, max_retries=5):\n", - "\n", - " state = _STATE.setdefault(keyword, {'until': until, 'rows': []})\n", - " \n", - " if state['until'] != until and state['rows']:\n", - " print(f\"Resuming '{keyword}' from until={state['until']}, {len(state['rows'])} posts already fetched this session.\")\n", - " \n", - " since_dt = datetime.fromisoformat(since.replace('Z', '+00:00'))\n", - " progress_bar = tqdm(total=limit, initial=min(len(state['rows']), limit), desc=\"Fetching posts\")\n", - "\n", + "def get_posts_by_keyword(keyword, since, until, limit=300):\n", " try:\n", - " while len(state['rows']) < limit:\n", - " try:\n", - " search_results = call_with_retry(client.app.bsky.feed.search_posts, {'q': keyword, 'since': since, 'until': state['until'], 'lang': 'en', 'limit': page_size}, max_retries=max_retries)\n", - " except Exception as e:\n", - " print(f\"Giving up on this page after {max_retries} retries: {e}\")\n", - " print(\"Progress so far is kept — just call this again to continue.\")\n", - " break\n", - " \n", + " num_posts = 0\n", + " post_data = []\n", + "\n", + " progress_bar = tqdm(total=limit, desc=\"Fetching posts\")\n", + "\n", + " while num_posts < limit:\n", + " \n", + " search_results = client.app.bsky.feed.search_posts({'q': keyword, 'since': since, 'until': until, 'lang': 'en', 'limit': 100})\n", " posts = search_results.posts\n", - " if not posts:\n", - " print(\"No more posts returned — reached the end of available results.\")\n", - " break\n", - " \n", - " oldest_date_obj = None\n", + "\n", " for post in posts:\n", - " result = extract_post(post)\n", - " if result is None:\n", - " continue\n", - " row, date_obj = result\n", - " state['rows'].append(row)\n", - " if oldest_date_obj is None or date_obj < oldest_date_obj:\n", - " oldest_date_obj = date_obj\n", - " \n", + " # Extract post details\n", + " text = clean_text(post.record.text) if hasattr(post.record, 'text') else np.nan\n", + " iso_date = post.record.created_at if hasattr(post.record, 'created_at') else np.nan\n", + " user = post.author.handle if hasattr(post.author, 'handle') else np.nan\n", + "\n", + " post_id = post.uri if hasattr(post, 'uri') else np.nan\n", + " parent_id = post.record.reply.parent.uri if hasattr(post.record, 'reply') and hasattr(post.record.reply, 'parent') else 'no_parent'\n", + "\n", + " # Format date-time\n", + " if iso_date != np.nan:\n", + " date_obj = datetime.fromisoformat(iso_date.replace('Z', '+00:00'))\n", + " date_str = date_obj.strftime('%Y-%m-%d %H:%M:%S')\n", + " else:\n", + " date_str = np.nan\n", + "\n", + " if post_id != np.nan and user != np.nan:\n", + " post_data.append({\n", + " 'date': date_str,\n", + " 'text': text,\n", + " 'user': user,\n", + " 'id': post_id,\n", + " 'parentid': parent_id\n", + " })\n", + " else:\n", + " continue\n", + " \n", + " until = iso_date\n", + " num_posts += len(posts)\n", " progress_bar.update(len(posts))\n", - " \n", - " if oldest_date_obj is not None:\n", - " state['until'] = oldest_date_obj.isoformat().replace('+00:00', 'Z')\n", - " progress_bar.set_postfix_str(f'now at: {state[\"until\"]}')\n", - " \n", - " if oldest_date_obj < since_dt + timedelta(hours=1):\n", - " print(\"Reached the `since` boundary.\")\n", - " break\n", - " \n", - " finally:\n", + " progress_bar.set_postfix_str(f'now at: {date_str}')\n", + "\n", + " if date_obj < (start_timestamp + timedelta(hours=1)):\n", + " break\n", + "\n", " progress_bar.close()\n", - " \n", - " print(f\"{len(state['rows'])} posts fetched in total this session for '{keyword}'. Call again with the same checkpoint_name to keep going.\")\n", - " return _dedupe(state['rows'])\n", + " df = pd.DataFrame(post_data)\n", + " return df\n", + "\n", + " except Exception as e:\n", + " print(f\"Error fetching posts: {e}\")\n", + " return pd.DataFrame()\n", "\n", "# Obtain posts from uri list\n", - "def get_posts(uris, chunk_size=20, max_retries=5):\n", - " state = _STATE.setdefault('_by_uri', {'done_uris': set(), 'rows': []})\n", - " \n", - " remaining = [u for u in uris if u not in state['done_uris']]\n", - " chunks = list(chunked(remaining, chunk_size))\n", - " \n", - " progress_bar = tqdm(total=len(uris), initial=len(state['done_uris']), desc=\"Fetching posts\")\n", - " \n", + "def get_posts(uris):\n", " try:\n", - " for uribit in chunks:\n", - " try:\n", - " search_results = call_with_retry(client.app.bsky.feed.get_posts, {'uris': uribit}, max_retries=max_retries)\n", - " except Exception as e:\n", - " print(f\"Skipping this chunk of {len(uribit)} URIs after {max_retries} retries: {e}\")\n", - " continue\n", - " \n", - " for post in search_results.posts:\n", - " result = extract_post(post)\n", - " if result is None:\n", + " post_data = []\n", + "\n", + " progress_bar = tqdm(total=len(uris), desc=\"Fetching posts\")\n", + "\n", + " for uribit in list(chunked(uris,20)):\n", + "\n", + " search_results = client.app.bsky.feed.get_posts({'uris': uribit})\n", + " posts = search_results.posts\n", + "\n", + " for post in posts: \n", + " # Extract post details\n", + " text = clean_text(post.record.text) if hasattr(post.record, 'text') else np.nan\n", + " iso_date = post.record.created_at if hasattr(post.record, 'created_at') else np.nan\n", + " user = post.author.handle if hasattr(post.author, 'handle') else np.nan\n", + "\n", + " post_id = post.uri if hasattr(post, 'uri') else np.nan\n", + " parent_id = post.record.reply.parent.uri if hasattr(post.record, 'reply') and hasattr(post.record.reply, 'parent') else 'no_parent'\n", + "\n", + " # Format date-time\n", + " if iso_date != np.nan:\n", + " date_obj = datetime.fromisoformat(iso_date.replace('Z', '+00:00'))\n", + " date_str = date_obj.strftime('%Y-%m-%d %H:%M:%S')\n", + " else:\n", + " date_str = np.nan\n", + "\n", + " if post_id != np.nan and user != np.nan:\n", + " post_data.append({\n", + " 'date': date_str,\n", + " 'text': text,\n", + " 'user': user,\n", + " 'id': post_id,\n", + " 'parentid': parent_id\n", + " })\n", + " else:\n", " continue\n", - " row, _ = result\n", - " state['rows'].append(row)\n", - " \n", - " state['done_uris'].update(uribit)\n", - " progress_bar.update(len(uribit))\n", - " \n", - " finally:\n", + " \n", + " progress_bar.update(len(posts))\n", + " \n", " progress_bar.close()\n", - " \n", - " print(f\"{len(state['done_uris'])}/{len(uris)} URIs processed in total.\")\n", - " return _dedupe(state['rows'])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Define state" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "_STATE = {}" + " df = pd.DataFrame(post_data)\n", + " return df\n", + "\n", + " except Exception as e:\n", + " print(f\"Error fetching posts: {e}\")\n", + " return pd.DataFrame()" ] }, { @@ -288,7 +233,7 @@ "metadata": {}, "outputs": [], "source": [ - "raw_df_data = get_posts_by_keyword('...', iso_start_timestamp, iso_end_timestamp, 100000)\n", + "raw_df_data = get_posts_by_keyword('...', iso_start_timestamp, iso_end_timestamp, 500000)\n", "raw_df_data.info()" ] }, @@ -306,19 +251,13 @@ "metadata": {}, "outputs": [], "source": [ - "_STATE = {}\n", "id_list = raw_df_data['id'].to_list()\n", "id_list.append('no_parent')\n", - "absent_parents = raw_df_data.loc[~raw_df_data['parentid'].isin(id_list), 'parentid'].tolist()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "\n", + "absent_parents = raw_df_data.loc[~raw_df_data['parentid'].isin(id_list), 'parentid'].tolist()\n", + "\n", "df_absent_data = get_posts(absent_parents)\n", + "\n", "df_absent_data.info()" ] }, @@ -336,7 +275,44 @@ "metadata": {}, "outputs": [], "source": [ - "df_data = pd.concat([raw_df_data, df_absent_data], ignore_index=True)\n", + "df_data = pd.concat(raw_df_data, df_absent_data, ignore_index=True)\n", + "\n", + "df_data.info()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Drop duplicates." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_data = df_data.drop_duplicates()\n", + "df_data.info()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Remove empty entries." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df_data.dropna(inplace=True,subset=['text'])\n", "df_data.info()" ] }, @@ -380,7 +356,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_data.to_csv(f'{case}.csv', index=False)" + "df_data.to_csv(f'../datasets/{case}.csv', index=False)" ] }, { @@ -482,7 +458,7 @@ "metadata": {}, "outputs": [], "source": [ - "closed_df_reduced_data.to_csv(f'{case}-social-closed.csv', index=False)" + "closed_df_reduced_data.to_csv(f'../datasets/{case}-social-closed.csv', index=False)" ] }, { @@ -499,7 +475,7 @@ "metadata": {}, "outputs": [], "source": [ - "closed_df_data.to_csv(f'{case}-closed.csv', index=False)" + "closed_df_data.to_csv(f'../datasets/{case}-closed.csv', index=False)" ] } ], diff --git a/graph-construct/construct-graph.ipynb b/graph-construct/construct-graph.ipynb index 4e89c69..31ea9fb 100644 --- a/graph-construct/construct-graph.ipynb +++ b/graph-construct/construct-graph.ipynb @@ -30,7 +30,7 @@ "case = '...'\n", "casegraphs = '...'\n", "\n", - "df_data = pd.read_csv(f'../sentiment-analysis/{case}.csv')\n", + "df_data = pd.read_csv(f'../datasets/{case}.csv')\n", "user_nodes = df_data.groupby('user')['id'].apply(list).to_dict()\n", "\n", "df_data.info()" @@ -62,7 +62,7 @@ " G.add_edge(row['id'], row['parentid'])\n", "\n", "# Export nearest-neighbour graph\n", - "nx.write_graphml(G, f'{casegraphs}-nn-graph.graphml')\n", + "nx.write_graphml(G, f'../datasets/{casegraphs}-nn-graph.graphml')\n", "\n", "# Define nearest-neighbour interaction count dictionary\n", "nn_count = {}\n", @@ -98,7 +98,7 @@ "tc_G = nx.transitive_closure(G)\n", "\n", "# Export transitive-closure graph\n", - "nx.write_graphml(tc_G, f'{casegraphs}-tc-graph.graphml')\n", + "nx.write_graphml(tc_G, f'../datasets/{casegraphs}-tc-graph.graphml')\n", "\n", "# Define transitive-closure interaction count dictionary\n", "tc_count = {}\n", @@ -139,19 +139,19 @@ " if row['parentid'] != 'no_parent':\n", " dG.add_edge(row['id'], row['parentid'])\n", "\n", - "nx.write_graphml(dG, f'{casegraphs}-nn-dgraph.graphml')\n", + "nx.write_graphml(dG, f'../datasets/{casegraphs}-nn-dgraph.graphml')\n", "\n", "# Construct directed-transitive-closure directed graph\n", "dtc_dG = nx.transitive_closure(dG)\n", "\n", "# Export directed-transitive-closure directed graph\n", - "nx.write_graphml(dtc_dG, f'{casegraphs}-dtc-dgraph.graphml')\n", + "nx.write_graphml(dtc_dG, f'../datasets/{casegraphs}-dtc-dgraph.graphml')\n", "\n", "# Convert to directed-transitive-closure undirected graph\n", "dtc_G = dtc_dG.to_undirected()\n", "\n", "# Export directed-transitive-closure graph\n", - "nx.write_graphml(dtc_G, f'{casegraphs}-dtc-graph.graphml')\n", + "nx.write_graphml(dtc_G, f'../datasets/{casegraphs}-dtc-graph.graphml')\n", "\n", "# Define directed-transitive-closure interaction count dictionary\n", "dtc_count = {}\n", @@ -196,7 +196,7 @@ " 'post-count': pd.Series(user_c)\n", "})\n", "\n", - "export_dataset = f'{case}-users.csv'\n", + "export_dataset = f'../datasets/{case}-users.csv'\n", "df_users.to_csv(export_dataset, index=False)\n", "\n", "df_users.info()" diff --git a/graph-inspect/inspect-graph-comparison.ipynb b/graph-inspect/inspect-graph-comparison.ipynb index b79049c..ea582af 100644 --- a/graph-inspect/inspect-graph-comparison.ipynb +++ b/graph-inspect/inspect-graph-comparison.ipynb @@ -33,8 +33,8 @@ "case2 = '...'\n", "\n", "# Import the graph\n", - "G1 = nx.read_graphml(f'../graph-construct/{case1}-dtc-graph.graphml')\n", - "G2 = nx.read_graphml(f'../graph-construct/{case2}-dtc-graph.graphml')\n", + "G1 = nx.read_graphml(f'../datasets/{case1}-dtc-graph.graphml')\n", + "G2 = nx.read_graphml(f'../datasets/{case2}-dtc-graph.graphml')\n", "\n", "# Print nodes, edges and density of the graphs\n", "print(f'G1:\\nNodes: {G1.number_of_nodes()}\\nEdges: {G1.number_of_edges()}\\nDensity: {nx.density(G1):.5f}')\n", diff --git a/sentiment-analysis/sentiment-analysis.py b/sentiment-analysis/sentiment-analysis.py index 3889cf8..3fa0094 100644 --- a/sentiment-analysis/sentiment-analysis.py +++ b/sentiment-analysis/sentiment-analysis.py @@ -1,56 +1,30 @@ import os import pandas as pd -from tqdm import tqdm import torch -from torch.utils.data import DataLoader, TensorDataset -from transformers import ( - DistilBertTokenizerFast, - DistilBertForSequenceClassification -) +from transformers import DistilBertTokenizerFast +from torch.utils.data import DataLoader +from transformers import DistilBertForSequenceClassification +from tqdm import tqdm # Importing the csv dataset. case = '...' -df_data = pd.read_csv(f'{case}.csv') +df_data = pd.read_csv(f'../datasets/{case}.csv') print(len(df_data)) -# Set device cuda -device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') -print('Using:', device) - -# Define model +# Preparing the dataset for the network this includes tokenization, encoding and creating dataloaders. model_name = 'distilbert-base-uncased' tokenizer = DistilBertTokenizerFast.from_pretrained(model_name) -# Tokenize and encode the text data -encodings = tokenizer( - df_data['text'].tolist(), - truncation=True, - padding=True, - return_tensors='pt' -) +encodings = tokenizer(df_data['text'].tolist(), truncation=True, padding=True, return_tensors='pt') -# Create tensor dataset -dataset = TensorDataset( +dataset = torch.utils.data.TensorDataset( encodings['input_ids'], encodings['attention_mask'] ) +dataloader = DataLoader(dataset, batch_size=45, shuffle=False) -# Create data loader -if device == 'cuda': - en_pin_memory = True -else: - en_pin_memory = False -dataloader = DataLoader( - dataset, - batch_size=53, - shuffle=False, - pin_memory=en_pin_memory -) - -# Loading the model -model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=1) -model.load_state_dict(torch.load('...', map_location=device)) -model.to(device) +# Loading the model. +model= torch.load('models/bert-aclimdb.pth') # Using the model to perform sentiment analysis on the dataset. model.eval() @@ -58,7 +32,7 @@ list_predicted_scores = [] for batch in tqdm(dataloader): with torch.no_grad(): - input_ids, attention_mask = [x.to(device) for x in batch] + input_ids, attention_mask = batch # Obtaining the sentiment score. output = model(input_ids=input_ids, attention_mask=attention_mask) @@ -71,4 +45,4 @@ for batch in tqdm(dataloader): df_data['s'] = list_predicted_scores # Exporting the dataframe to a csv dataset. -df_data.to_csv(f'{case}-s.csv', index=False) +df_data.to_csv(f'../datasets/{case}-s.csv', index=False) \ No newline at end of file diff --git a/sentiment-analysis/training/aclimdb-sentiment-training.ipynb b/sentiment-analysis/training/aclimdb-sentiment-training.ipynb index 312e19c..31547a4 100644 --- a/sentiment-analysis/training/aclimdb-sentiment-training.ipynb +++ b/sentiment-analysis/training/aclimdb-sentiment-training.ipynb @@ -27,8 +27,8 @@ "import os\n", "import pandas as pd\n", "\n", - "imdb_train_dataset = 'aclimdb/train'\n", - "imdb_test_dataset = 'aclimdb/test'\n", + "imdb_train_dataset = \"aclimdb/train\"\n", + "imdb_test_dataset = \"aclimdb/test\"\n", "\n", "train_reviews = []\n", "train_scores = []\n", @@ -118,7 +118,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(f'Length training set: {len(df_train)}\\nLength testing set: {len(df_test)}')" + "print(f\"Length training set: {len(df_train)}\\nLength testing set: {len(df_test)}\")" ] }, { @@ -136,67 +136,31 @@ "outputs": [], "source": [ "import torch\n", - "from torch.utils.data import DataLoader, TensorDataset\n", - "from transformers import (\n", - " DistilBertTokenizerFast,\n", - " DistilBertForSequenceClassification\n", - ")\n", + "from transformers import DistilBertTokenizerFast\n", + "from torch.utils.data import DataLoader\n", + "from transformers import DistilBertForSequenceClassification\n", "\n", - "# Set device cuda\n", - "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", - "print('Using:', device)\n", - "\n", - "# Define model\n", - "model_name = 'distilbert-base-uncased'\n", + "model_name = \"distilbert-base-uncased\"\n", "tokenizer = DistilBertTokenizerFast.from_pretrained(model_name)\n", "\n", "# Tokenize and encode the text data\n", - "train_encodings = tokenizer(\n", - " df_train['text'].tolist(), \n", - " truncation=True, \n", - " padding=True, \n", - " return_tensors='pt'\n", - ")\n", + "train_encodings = tokenizer(df_train['text'].tolist(), truncation=True, padding=True, return_tensors='pt')\n", + "test_encodings = tokenizer(df_test['text'].tolist(), truncation=True, padding=True, return_tensors='pt')\n", "\n", - "test_encodings = tokenizer(\n", - " df_test['text'].tolist(), \n", - " truncation=True, \n", - " padding=True, \n", - " return_tensors='pt'\n", - ")\n", - "\n", - "# Create tensor dataset\n", - "train_dataset = TensorDataset(\n", + "# Create data loaders\n", + "train_dataset = torch.utils.data.TensorDataset(\n", " train_encodings['input_ids'], \n", " train_encodings['attention_mask'], \n", " torch.tensor(df_train['s'].values, dtype=torch.float32)\n", ")\n", + "train_dataloader = DataLoader(train_dataset, batch_size=25, shuffle=True)\n", "\n", - "test_dataset = TensorDataset(\n", + "test_dataset = torch.utils.data.TensorDataset(\n", " test_encodings['input_ids'], \n", " test_encodings['attention_mask'], \n", " torch.tensor(df_test['s'].values, dtype=torch.float32)\n", ")\n", - "\n", - "# Create data loaders\n", - "if device == 'cuda':\n", - " en_pin_memory = True\n", - "else:\n", - " en_pin_memory = False\n", - "\n", - "train_dataloader = DataLoader(\n", - " train_dataset, \n", - " batch_size=10, \n", - " shuffle=True,\n", - " pin_memory=en_pin_memory\n", - ")\n", - "\n", - "test_dataloader = DataLoader(\n", - " test_dataset, \n", - " batch_size=10, \n", - " shuffle=False,\n", - " pin_memory=en_pin_memory\n", - ")" + "test_dataloader = DataLoader(test_dataset, batch_size=25, shuffle=False)" ] }, { @@ -213,10 +177,7 @@ "metadata": {}, "outputs": [], "source": [ - "model = DistilBertForSequenceClassification.from_pretrained(\n", - " model_name, \n", - " num_labels=1\n", - ").to(device)" + "model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=1)" ] }, { @@ -274,7 +235,7 @@ " for batch in train_dataloader:\n", " global_step += 1\n", " num_batches += 1\n", - " input_ids, attention_mask, target_scores = [x.to(device) for x in batch]\n", + " input_ids, attention_mask, target_scores = batch\n", "\n", " # Forward pass\n", " output = model(input_ids=input_ids, attention_mask=attention_mask)\n", @@ -309,7 +270,7 @@ "\n", " for batch in test_dataloader:\n", " with torch.no_grad():\n", - " input_ids, attention_mask, target_scores = [x.to(device) for x in batch]\n", + " input_ids, attention_mask, target_scores = batch\n", "\n", " # Obtaining the scores\n", " output = model(input_ids=input_ids, attention_mask=attention_mask)\n", @@ -329,7 +290,7 @@ " print(f\"Epoch {epoch + 1}/{num_epochs}, Validation loss: {average_loss:.4f}\\n\")\n", "\n", " # Saving the model\n", - " torch.save(model.state_dict(), f'bert-aclimdb/{epoch + 1}.pth')\n", + " torch.save(model, f'bert-aclimdb/{epoch + 1}.pth')\n", "\n", " # Early stopping check\n", " if average_loss < best_validation_loss:\n", @@ -339,7 +300,6 @@ " no_improvement_counter += 1\n", "\n", " if no_improvement_counter >= early_stop_patience:\n", - " print('Early stopping triggered')\n", " break\n", "\n", "writer.close()" @@ -359,9 +319,7 @@ "metadata": {}, "outputs": [], "source": [ - "model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=1)\n", - "model.load_state_dict(torch.load('...', map_location=device))\n", - "model.to(device)" + "model = torch.load('bert-aclimdb/4.pth')" ] }, { @@ -386,7 +344,7 @@ "\n", "for batch in test_dataloader:\n", " with torch.no_grad():\n", - " input_ids, attention_mask, scores = [x.to(device) for x in batch]\n", + " input_ids, attention_mask, scores = batch\n", "\n", " # Obtaining the scores\n", " output = model(input_ids=input_ids, attention_mask=attention_mask)\n",