diff --git "a/notebooks/05c_newsScraper_clearning.ipynb" "b/notebooks/05c_newsScraper_clearning.ipynb"
new file mode 100644--- /dev/null
+++ "b/notebooks/05c_newsScraper_clearning.ipynb"
@@ -0,0 +1,1931 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3ddf645f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "42801c6a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "workding dir: d:\\code\\projects\\global-incidents\n",
+      "loading env vars from: d:\\code\\projects\\global-incidents\\.env.example\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "if \"workding_dir\" not in globals():\n",
+    "    workding_dir = str(Path.cwd().parent)\n",
+    "\n",
+    "os.chdir(workding_dir)\n",
+    "sys.path.append(workding_dir)\n",
+    "print(\"workding dir:\", workding_dir)\n",
+    "\n",
+    "from dotenv import find_dotenv, load_dotenv\n",
+    "\n",
+    "found_dotenv = find_dotenv(\".env\")\n",
+    "\n",
+    "if len(found_dotenv) == 0:\n",
+    "    found_dotenv = find_dotenv(\".env.example\")\n",
+    "print(f\"loading env vars from: {found_dotenv}\")\n",
+    "load_dotenv(found_dotenv, override=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "ddf1e32e-7751-43db-9b5a-22cb08e35c6c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "\n",
+    "df = pd.read_csv(\"data/scrapped_data2.csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "a9c6ed07-a7d0-4aaa-b571-038919c75e05",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>Headline</th>\n",
+       "      <th>Details</th>\n",
+       "      <th>Severity</th>\n",
+       "      <th>Category</th>\n",
+       "      <th>Region</th>\n",
+       "      <th>Datetime</th>\n",
+       "      <th>Year</th>\n",
+       "      <th>lat</th>\n",
+       "      <th>lon</th>\n",
+       "      <th>maritime_label</th>\n",
+       "      <th>found_ports</th>\n",
+       "      <th>contains_port_info</th>\n",
+       "      <th>if_labeled</th>\n",
+       "      <th>Month</th>\n",
+       "      <th>Week</th>\n",
+       "      <th>Headline_Details</th>\n",
+       "      <th>url</th>\n",
+       "      <th>title</th>\n",
+       "      <th>content</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>Media sources indicate that workers at the Gra...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Mine Workers Strike</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>28/5/17 17:08</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>-4.05608</td>\n",
+       "      <td>137.11302</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['freeport']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiZ2h0d...</td>\n",
+       "      <td>Freeport Indonesia mine workers extend strike ...</td>\n",
+       "      <td>Trucks are seen on a road in the Grasberg copp...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2</td>\n",
+       "      <td>Indonesia: Undersea internet cables damaged by...</td>\n",
+       "      <td>News sources are stating that recent typhoons ...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Travel Warning</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>4/9/17 14:30</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['hong kong']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>14.0</td>\n",
+       "      <td>Indonesia: Undersea internet cables damaged by...</td>\n",
+       "      <td>no url found</td>\n",
+       "      <td>no title found</td>\n",
+       "      <td>no content found</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>3</td>\n",
+       "      <td>Shanghai port congestion impacts terminals in ...</td>\n",
+       "      <td>The persisting port congestion at Shanghai’s Y...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Port Congestion</td>\n",
+       "      <td>China</td>\n",
+       "      <td>27/4/17 9:16</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>29.52000</td>\n",
+       "      <td>121.33190</td>\n",
+       "      <td>True</td>\n",
+       "      <td>['ningbo', 'qingdao', 'shanghai']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>17.0</td>\n",
+       "      <td>Shanghai port congestion impacts terminals in ...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiVWh0d...</td>\n",
+       "      <td>Typhoon Muifa to shut China ports for second t...</td>\n",
+       "      <td>By Sam Whelan 13/09/2022\\n\\nAnother typhoon ha...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>4</td>\n",
+       "      <td>UPDATE - Indonesia: Explosion at KP Terminal i...</td>\n",
+       "      <td>Updated local media sources from Jakarta indic...</td>\n",
+       "      <td>Extreme</td>\n",
+       "      <td>Bombing, Police Operations</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>24/5/17 15:15</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>-6.22465</td>\n",
+       "      <td>106.86700</td>\n",
+       "      <td>True</td>\n",
+       "      <td>['jakarta']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>UPDATE - Indonesia: Explosion at KP Terminal i...</td>\n",
+       "      <td>no url found</td>\n",
+       "      <td>no title found</td>\n",
+       "      <td>no content found</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>5</td>\n",
+       "      <td>UPDATE - Indonesia: Police confirm two explosi...</td>\n",
+       "      <td>According to local police in Jakarta, two expl...</td>\n",
+       "      <td>Extreme</td>\n",
+       "      <td>Bombing, Police Operations</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>24/5/17 16:20</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>True</td>\n",
+       "      <td>['jakarta']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>True</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>UPDATE - Indonesia: Police confirm two explosi...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiZWh0d...</td>\n",
+       "      <td>Jakarta Police Receive 2 More Reports on Coldp...</td>\n",
+       "      <td>TEMPO.CO, Jakarta - South Jakarta Metro Police...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5705</th>\n",
+       "      <td>5776</td>\n",
+       "      <td>Winter storm may bring inclement weather condi...</td>\n",
+       "      <td>Intelligence received by Everstream Analytics ...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Ice Storm</td>\n",
+       "      <td>United States</td>\n",
+       "      <td>16/12/20 2:47</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>38.90072</td>\n",
+       "      <td>-77.05440</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['new york']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>12.0</td>\n",
+       "      <td>51.0</td>\n",
+       "      <td>Winter storm may bring inclement weather condi...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMia2h0d...</td>\n",
+       "      <td>Big storm to dump heavy rain and snow along Ea...</td>\n",
+       "      <td>A sprawling winter storm known as a nor’easter...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5706</th>\n",
+       "      <td>5777</td>\n",
+       "      <td>Winter weather expected to continue to impact ...</td>\n",
+       "      <td>Meteorological sources indicate that a series ...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Roadway Closure / Disruption, Ground Transport...</td>\n",
+       "      <td>United States</td>\n",
+       "      <td>31/12/20 18:15</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>41.30357</td>\n",
+       "      <td>-72.90561</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['new york', 'virginia']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>True</td>\n",
+       "      <td>12.0</td>\n",
+       "      <td>53.0</td>\n",
+       "      <td>Winter weather expected to continue to impact ...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiZ2h0d...</td>\n",
+       "      <td>Cross-country storm brings severe weather thre...</td>\n",
+       "      <td>A powerful cross-country storm that impacted t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5707</th>\n",
+       "      <td>5778</td>\n",
+       "      <td>Workers of Svitzer Australia plan to strike on...</td>\n",
+       "      <td>Industry sources report on December 7 that Svi...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Industrial Action</td>\n",
+       "      <td>Australia</td>\n",
+       "      <td>7/12/20 6:16</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>-38.35169</td>\n",
+       "      <td>145.25050</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['geelong', 'melbourne']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>7.0</td>\n",
+       "      <td>28.0</td>\n",
+       "      <td>Workers of Svitzer Australia plan to strike on...</td>\n",
+       "      <td>no url found</td>\n",
+       "      <td>no title found</td>\n",
+       "      <td>no content found</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5708</th>\n",
+       "      <td>5779</td>\n",
+       "      <td>Workers stage 24-hour stoppage at DP World Ter...</td>\n",
+       "      <td>Industry sources indicate on December 14 that ...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Port Strike</td>\n",
+       "      <td>Australia</td>\n",
+       "      <td>14/12/20 16:52</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>True</td>\n",
+       "      <td>['fremantle']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>True</td>\n",
+       "      <td>12.0</td>\n",
+       "      <td>51.0</td>\n",
+       "      <td>Workers stage 24-hour stoppage at DP World Ter...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiOWh0d...</td>\n",
+       "      <td>Union shuts down strikes as DP World threatens...</td>\n",
+       "      <td>The Maritime Union of Australia (MUA) has curt...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5709</th>\n",
+       "      <td>5780</td>\n",
+       "      <td>Workers with Ertsoverslagbedrijf Europoort C.V...</td>\n",
+       "      <td>On November 17, Dutch media sources reported t...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Port Strike</td>\n",
+       "      <td>Netherlands</td>\n",
+       "      <td>18/11/20 17:55</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>51.96121</td>\n",
+       "      <td>4.10550</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['rotterdam']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>True</td>\n",
+       "      <td>11.0</td>\n",
+       "      <td>47.0</td>\n",
+       "      <td>Workers with Ertsoverslagbedrijf Europoort C.V...</td>\n",
+       "      <td>no url found</td>\n",
+       "      <td>no title found</td>\n",
+       "      <td>no content found</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5710 rows × 20 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        id                                           Headline  \\\n",
+       "0        1  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "1        2  Indonesia: Undersea internet cables damaged by...   \n",
+       "2        3  Shanghai port congestion impacts terminals in ...   \n",
+       "3        4  UPDATE - Indonesia: Explosion at KP Terminal i...   \n",
+       "4        5  UPDATE - Indonesia: Police confirm two explosi...   \n",
+       "...    ...                                                ...   \n",
+       "5705  5776  Winter storm may bring inclement weather condi...   \n",
+       "5706  5777  Winter weather expected to continue to impact ...   \n",
+       "5707  5778  Workers of Svitzer Australia plan to strike on...   \n",
+       "5708  5779  Workers stage 24-hour stoppage at DP World Ter...   \n",
+       "5709  5780  Workers with Ertsoverslagbedrijf Europoort C.V...   \n",
+       "\n",
+       "                                                Details  Severity  \\\n",
+       "0     Media sources indicate that workers at the Gra...  Moderate   \n",
+       "1     News sources are stating that recent typhoons ...     Minor   \n",
+       "2     The persisting port congestion at Shanghai’s Y...     Minor   \n",
+       "3     Updated local media sources from Jakarta indic...   Extreme   \n",
+       "4     According to local police in Jakarta, two expl...   Extreme   \n",
+       "...                                                 ...       ...   \n",
+       "5705  Intelligence received by Everstream Analytics ...  Moderate   \n",
+       "5706  Meteorological sources indicate that a series ...     Minor   \n",
+       "5707  Industry sources report on December 7 that Svi...  Moderate   \n",
+       "5708  Industry sources indicate on December 14 that ...  Moderate   \n",
+       "5709  On November 17, Dutch media sources reported t...     Minor   \n",
+       "\n",
+       "                                               Category         Region  \\\n",
+       "0                                   Mine Workers Strike      Indonesia   \n",
+       "1                                        Travel Warning      Indonesia   \n",
+       "2                                       Port Congestion          China   \n",
+       "3                            Bombing, Police Operations      Indonesia   \n",
+       "4                            Bombing, Police Operations      Indonesia   \n",
+       "...                                                 ...            ...   \n",
+       "5705                                          Ice Storm  United States   \n",
+       "5706  Roadway Closure / Disruption, Ground Transport...  United States   \n",
+       "5707                                  Industrial Action      Australia   \n",
+       "5708                                        Port Strike      Australia   \n",
+       "5709                                        Port Strike    Netherlands   \n",
+       "\n",
+       "            Datetime    Year       lat        lon  maritime_label  \\\n",
+       "0      28/5/17 17:08  2017.0  -4.05608  137.11302           False   \n",
+       "1       4/9/17 14:30  2017.0       NaN        NaN           False   \n",
+       "2       27/4/17 9:16  2017.0  29.52000  121.33190            True   \n",
+       "3      24/5/17 15:15  2017.0  -6.22465  106.86700            True   \n",
+       "4      24/5/17 16:20  2017.0       NaN        NaN            True   \n",
+       "...              ...     ...       ...        ...             ...   \n",
+       "5705   16/12/20 2:47  2020.0  38.90072  -77.05440           False   \n",
+       "5706  31/12/20 18:15  2020.0  41.30357  -72.90561           False   \n",
+       "5707    7/12/20 6:16  2020.0 -38.35169  145.25050           False   \n",
+       "5708  14/12/20 16:52  2020.0       NaN        NaN            True   \n",
+       "5709  18/11/20 17:55  2020.0  51.96121    4.10550           False   \n",
+       "\n",
+       "                            found_ports  contains_port_info if_labeled  Month  \\\n",
+       "0                          ['freeport']                 1.0      False    5.0   \n",
+       "1                         ['hong kong']                 1.0      False    4.0   \n",
+       "2     ['ningbo', 'qingdao', 'shanghai']                 1.0      False    4.0   \n",
+       "3                           ['jakarta']                 1.0      False    5.0   \n",
+       "4                           ['jakarta']                 1.0       True    5.0   \n",
+       "...                                 ...                 ...        ...    ...   \n",
+       "5705                       ['new york']                 1.0      False   12.0   \n",
+       "5706           ['new york', 'virginia']                 1.0       True   12.0   \n",
+       "5707           ['geelong', 'melbourne']                 1.0      False    7.0   \n",
+       "5708                      ['fremantle']                 1.0       True   12.0   \n",
+       "5709                      ['rotterdam']                 1.0       True   11.0   \n",
+       "\n",
+       "      Week                                   Headline_Details  \\\n",
+       "0     21.0  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "1     14.0  Indonesia: Undersea internet cables damaged by...   \n",
+       "2     17.0  Shanghai port congestion impacts terminals in ...   \n",
+       "3     21.0  UPDATE - Indonesia: Explosion at KP Terminal i...   \n",
+       "4     21.0  UPDATE - Indonesia: Police confirm two explosi...   \n",
+       "...    ...                                                ...   \n",
+       "5705  51.0  Winter storm may bring inclement weather condi...   \n",
+       "5706  53.0  Winter weather expected to continue to impact ...   \n",
+       "5707  28.0  Workers of Svitzer Australia plan to strike on...   \n",
+       "5708  51.0  Workers stage 24-hour stoppage at DP World Ter...   \n",
+       "5709  47.0  Workers with Ertsoverslagbedrijf Europoort C.V...   \n",
+       "\n",
+       "                                                    url  \\\n",
+       "0     https://news.google.com/rss/articles/CBMiZ2h0d...   \n",
+       "1                                          no url found   \n",
+       "2     https://news.google.com/rss/articles/CBMiVWh0d...   \n",
+       "3                                          no url found   \n",
+       "4     https://news.google.com/rss/articles/CBMiZWh0d...   \n",
+       "...                                                 ...   \n",
+       "5705  https://news.google.com/rss/articles/CBMia2h0d...   \n",
+       "5706  https://news.google.com/rss/articles/CBMiZ2h0d...   \n",
+       "5707                                       no url found   \n",
+       "5708  https://news.google.com/rss/articles/CBMiOWh0d...   \n",
+       "5709                                       no url found   \n",
+       "\n",
+       "                                                  title  \\\n",
+       "0     Freeport Indonesia mine workers extend strike ...   \n",
+       "1                                        no title found   \n",
+       "2     Typhoon Muifa to shut China ports for second t...   \n",
+       "3                                        no title found   \n",
+       "4     Jakarta Police Receive 2 More Reports on Coldp...   \n",
+       "...                                                 ...   \n",
+       "5705  Big storm to dump heavy rain and snow along Ea...   \n",
+       "5706  Cross-country storm brings severe weather thre...   \n",
+       "5707                                     no title found   \n",
+       "5708  Union shuts down strikes as DP World threatens...   \n",
+       "5709                                     no title found   \n",
+       "\n",
+       "                                                content  \n",
+       "0     Trucks are seen on a road in the Grasberg copp...  \n",
+       "1                                      no content found  \n",
+       "2     By Sam Whelan 13/09/2022\\n\\nAnother typhoon ha...  \n",
+       "3                                      no content found  \n",
+       "4     TEMPO.CO, Jakarta - South Jakarta Metro Police...  \n",
+       "...                                                 ...  \n",
+       "5705  A sprawling winter storm known as a nor’easter...  \n",
+       "5706  A powerful cross-country storm that impacted t...  \n",
+       "5707                                   no content found  \n",
+       "5708  The Maritime Union of Australia (MUA) has curt...  \n",
+       "5709                                   no content found  \n",
+       "\n",
+       "[5710 rows x 20 columns]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "6120344f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "url\n",
+       "no url found                                                                                                                                                                                                                                                  1945\n",
+       "cannot scrape the url                                                                                                                                                                                                                                           71\n",
+       "https://news.google.com/rss/articles/CBMiVWh0dHBzOi8vd3d3Lm1hZXJzay5jb20vbmV3cy9hcnRpY2xlcy8yMDIzLzA4LzIyL2xhdGluLWFtZXJpY2EtbWFya2V0LXVwZGF0ZS1hdWd1c3QtMjPSAQA?oc=5&hl=en-SG&gl=SG&ceid=SG:en                                                                 39\n",
+       "https://news.google.com/rss/articles/CBMia2h0dHBzOi8vd3d3LmNoaW5hLWJyaWVmaW5nLmNvbS9uZXdzL2NoaW5hLWNvcm9uYXZpcnVzLXVwZGF0ZXMtbGF0ZXN0LWRldmVsb3BtZW50cy1idXNpbmVzcy1hZHZpc29yeS1wYXJ0LTIv0gEA?oc=5&hl=en-SG&gl=SG&ceid=SG:en                                    38\n",
+       "https://news.google.com/rss/articles/CBMigAFodHRwczovL3d3dy5ldXJvbmV3cy5jb20vdHJhdmVsLzIwMjQvMDMvMDYvZXVyb3Blcy1zdW1tZXItdHJhdmVsLXN0cmlrZXMtd2hlbi13aGVyZS1hbmQtd2hhdC1kaXNydXB0aW9uLXlvdS1jYW4tZXhwZWN0LWluLWF1Z3VzdNIBAA?oc=5&hl=en-SG&gl=SG&ceid=SG:en      35\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.url.value_counts()[:5]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "0acaf899-cd96-4424-b825-c71bf042355a",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>Headline</th>\n",
+       "      <th>Details</th>\n",
+       "      <th>Severity</th>\n",
+       "      <th>Category</th>\n",
+       "      <th>Region</th>\n",
+       "      <th>Datetime</th>\n",
+       "      <th>Year</th>\n",
+       "      <th>lat</th>\n",
+       "      <th>lon</th>\n",
+       "      <th>maritime_label</th>\n",
+       "      <th>found_ports</th>\n",
+       "      <th>contains_port_info</th>\n",
+       "      <th>if_labeled</th>\n",
+       "      <th>Month</th>\n",
+       "      <th>Week</th>\n",
+       "      <th>Headline_Details</th>\n",
+       "      <th>url</th>\n",
+       "      <th>title</th>\n",
+       "      <th>content</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>Media sources indicate that workers at the Gra...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Mine Workers Strike</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>28/5/17 17:08</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>-4.05608</td>\n",
+       "      <td>137.11302</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['freeport']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>Grasberg Mine- Grasberg mine workers extend st...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiZ2h0d...</td>\n",
+       "      <td>Freeport Indonesia mine workers extend strike ...</td>\n",
+       "      <td>Trucks are seen on a road in the Grasberg copp...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>3</td>\n",
+       "      <td>Shanghai port congestion impacts terminals in ...</td>\n",
+       "      <td>The persisting port congestion at Shanghai’s Y...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Port Congestion</td>\n",
+       "      <td>China</td>\n",
+       "      <td>27/4/17 9:16</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>29.52000</td>\n",
+       "      <td>121.33190</td>\n",
+       "      <td>True</td>\n",
+       "      <td>['ningbo', 'qingdao', 'shanghai']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>17.0</td>\n",
+       "      <td>Shanghai port congestion impacts terminals in ...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiVWh0d...</td>\n",
+       "      <td>Typhoon Muifa to shut China ports for second t...</td>\n",
+       "      <td>By Sam Whelan 13/09/2022\\n\\nAnother typhoon ha...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>5</td>\n",
+       "      <td>UPDATE - Indonesia: Police confirm two explosi...</td>\n",
+       "      <td>According to local police in Jakarta, two expl...</td>\n",
+       "      <td>Extreme</td>\n",
+       "      <td>Bombing, Police Operations</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>24/5/17 16:20</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>True</td>\n",
+       "      <td>['jakarta']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>True</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>21.0</td>\n",
+       "      <td>UPDATE - Indonesia: Police confirm two explosi...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiZWh0d...</td>\n",
+       "      <td>Jakarta Police Receive 2 More Reports on Coldp...</td>\n",
+       "      <td>TEMPO.CO, Jakarta - South Jakarta Metro Police...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>6</td>\n",
+       "      <td>UPDATE - Indonesia: Severe winds damage infras...</td>\n",
+       "      <td>Severe winds have downed billboards and trees ...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Roadway Closure / Disruption, Flooding, Severe...</td>\n",
+       "      <td>Indonesia</td>\n",
+       "      <td>19/4/17 9:10</td>\n",
+       "      <td>2017.0</td>\n",
+       "      <td>-6.91264</td>\n",
+       "      <td>107.65700</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['jakarta']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>True</td>\n",
+       "      <td>4.0</td>\n",
+       "      <td>16.0</td>\n",
+       "      <td>UPDATE - Indonesia: Severe winds damage infras...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiSWh0d...</td>\n",
+       "      <td>Indonesia hit by some of strongest winds recorded</td>\n",
+       "      <td>A man stands near damaged houses following a t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>14</td>\n",
+       "      <td>2 miles E of Chesterfield - A tornado has touc...</td>\n",
+       "      <td>Government sources are reporting a tornado has...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Tornado</td>\n",
+       "      <td>United States</td>\n",
+       "      <td>17/9/18 19:55</td>\n",
+       "      <td>2018.0</td>\n",
+       "      <td>37.51000</td>\n",
+       "      <td>-77.61000</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['virginia']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>True</td>\n",
+       "      <td>9.0</td>\n",
+       "      <td>38.0</td>\n",
+       "      <td>2 miles E of Chesterfield - A tornado has touc...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMigAFod...</td>\n",
+       "      <td>UPDATE: Number of homes without power down to ...</td>\n",
+       "      <td>More than 90,000 homes and businesses across t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5699</th>\n",
+       "      <td>5770</td>\n",
+       "      <td>Wildfire started by lightning strike near Cart...</td>\n",
+       "      <td>Local media sources are reporting a wildfire o...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Wildfire</td>\n",
+       "      <td>Spain</td>\n",
+       "      <td>27/11/20 11:19</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>37.59520</td>\n",
+       "      <td>-1.10078</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['cartagena']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>11.0</td>\n",
+       "      <td>48.0</td>\n",
+       "      <td>Wildfire started by lightning strike near Cart...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMibmh0d...</td>\n",
+       "      <td>Neanderthals Were As Smart As Modern Humans an...</td>\n",
+       "      <td>Representational image (gorodenkoff, Getty Ima...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5700</th>\n",
+       "      <td>5771</td>\n",
+       "      <td>Wind Event to Impact Southern CA This Week; Da...</td>\n",
+       "      <td>Meteorologists are predicting a new round of S...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Weather Advisory, Hazmat Response, Network Dis...</td>\n",
+       "      <td>United States</td>\n",
+       "      <td>1/12/20 0:08</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['los angeles']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>True</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>2.0</td>\n",
+       "      <td>Wind Event to Impact Southern CA This Week; Da...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMihgFod...</td>\n",
+       "      <td>Hurricane Hilary live updates: Newsom declares...</td>\n",
+       "      <td>Staff reports\\n\\nPalm Springs Desert Sun\\n\\nLo...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5701</th>\n",
+       "      <td>5772</td>\n",
+       "      <td>Wind stoppages expected for the Port of Durban...</td>\n",
+       "      <td>Shipping sources indicate on November 11 that ...</td>\n",
+       "      <td>Minor</td>\n",
+       "      <td>Port Disruption</td>\n",
+       "      <td>South Africa</td>\n",
+       "      <td>11/11/20 16:03</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['durban']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>True</td>\n",
+       "      <td>11.0</td>\n",
+       "      <td>46.0</td>\n",
+       "      <td>Wind stoppages expected for the Port of Durban...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMiigFod...</td>\n",
+       "      <td>Severe thunderstorms expected in most parts of...</td>\n",
+       "      <td>DURBAN - Severe thunderstorms are expected in ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5702</th>\n",
+       "      <td>5773</td>\n",
+       "      <td>Wind warnings issue for Melbourne coast for Oc...</td>\n",
+       "      <td>Media sources reported that wind warnings for ...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Weather Advisory</td>\n",
+       "      <td>Australia</td>\n",
+       "      <td>15/10/20 6:19</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>-37.84796</td>\n",
+       "      <td>144.94660</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['melbourne']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>10.0</td>\n",
+       "      <td>42.0</td>\n",
+       "      <td>Wind warnings issue for Melbourne coast for Oc...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMieGh0d...</td>\n",
+       "      <td>NRL Draw 2023: All the scores, results and sch...</td>\n",
+       "      <td>The 2023 NRL season has finally come to an end...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5705</th>\n",
+       "      <td>5776</td>\n",
+       "      <td>Winter storm may bring inclement weather condi...</td>\n",
+       "      <td>Intelligence received by Everstream Analytics ...</td>\n",
+       "      <td>Moderate</td>\n",
+       "      <td>Ice Storm</td>\n",
+       "      <td>United States</td>\n",
+       "      <td>16/12/20 2:47</td>\n",
+       "      <td>2020.0</td>\n",
+       "      <td>38.90072</td>\n",
+       "      <td>-77.05440</td>\n",
+       "      <td>False</td>\n",
+       "      <td>['new york']</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>False</td>\n",
+       "      <td>12.0</td>\n",
+       "      <td>51.0</td>\n",
+       "      <td>Winter storm may bring inclement weather condi...</td>\n",
+       "      <td>https://news.google.com/rss/articles/CBMia2h0d...</td>\n",
+       "      <td>Big storm to dump heavy rain and snow along Ea...</td>\n",
+       "      <td>A sprawling winter storm known as a nor’easter...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>2786 rows × 20 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        id                                           Headline  \\\n",
+       "0        1  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "2        3  Shanghai port congestion impacts terminals in ...   \n",
+       "4        5  UPDATE - Indonesia: Police confirm two explosi...   \n",
+       "5        6  UPDATE - Indonesia: Severe winds damage infras...   \n",
+       "13      14  2 miles E of Chesterfield - A tornado has touc...   \n",
+       "...    ...                                                ...   \n",
+       "5699  5770  Wildfire started by lightning strike near Cart...   \n",
+       "5700  5771  Wind Event to Impact Southern CA This Week; Da...   \n",
+       "5701  5772  Wind stoppages expected for the Port of Durban...   \n",
+       "5702  5773  Wind warnings issue for Melbourne coast for Oc...   \n",
+       "5705  5776  Winter storm may bring inclement weather condi...   \n",
+       "\n",
+       "                                                Details  Severity  \\\n",
+       "0     Media sources indicate that workers at the Gra...  Moderate   \n",
+       "2     The persisting port congestion at Shanghai’s Y...     Minor   \n",
+       "4     According to local police in Jakarta, two expl...   Extreme   \n",
+       "5     Severe winds have downed billboards and trees ...  Moderate   \n",
+       "13    Government sources are reporting a tornado has...     Minor   \n",
+       "...                                                 ...       ...   \n",
+       "5699  Local media sources are reporting a wildfire o...     Minor   \n",
+       "5700  Meteorologists are predicting a new round of S...  Moderate   \n",
+       "5701  Shipping sources indicate on November 11 that ...     Minor   \n",
+       "5702  Media sources reported that wind warnings for ...  Moderate   \n",
+       "5705  Intelligence received by Everstream Analytics ...  Moderate   \n",
+       "\n",
+       "                                               Category         Region  \\\n",
+       "0                                   Mine Workers Strike      Indonesia   \n",
+       "2                                       Port Congestion          China   \n",
+       "4                            Bombing, Police Operations      Indonesia   \n",
+       "5     Roadway Closure / Disruption, Flooding, Severe...      Indonesia   \n",
+       "13                                              Tornado  United States   \n",
+       "...                                                 ...            ...   \n",
+       "5699                                           Wildfire          Spain   \n",
+       "5700  Weather Advisory, Hazmat Response, Network Dis...  United States   \n",
+       "5701                                    Port Disruption   South Africa   \n",
+       "5702                                   Weather Advisory      Australia   \n",
+       "5705                                          Ice Storm  United States   \n",
+       "\n",
+       "            Datetime    Year       lat        lon  maritime_label  \\\n",
+       "0      28/5/17 17:08  2017.0  -4.05608  137.11302           False   \n",
+       "2       27/4/17 9:16  2017.0  29.52000  121.33190            True   \n",
+       "4      24/5/17 16:20  2017.0       NaN        NaN            True   \n",
+       "5       19/4/17 9:10  2017.0  -6.91264  107.65700           False   \n",
+       "13     17/9/18 19:55  2018.0  37.51000  -77.61000           False   \n",
+       "...              ...     ...       ...        ...             ...   \n",
+       "5699  27/11/20 11:19  2020.0  37.59520   -1.10078           False   \n",
+       "5700    1/12/20 0:08  2020.0       NaN        NaN           False   \n",
+       "5701  11/11/20 16:03  2020.0       NaN        NaN           False   \n",
+       "5702   15/10/20 6:19  2020.0 -37.84796  144.94660           False   \n",
+       "5705   16/12/20 2:47  2020.0  38.90072  -77.05440           False   \n",
+       "\n",
+       "                            found_ports  contains_port_info if_labeled  Month  \\\n",
+       "0                          ['freeport']                 1.0      False    5.0   \n",
+       "2     ['ningbo', 'qingdao', 'shanghai']                 1.0      False    4.0   \n",
+       "4                           ['jakarta']                 1.0       True    5.0   \n",
+       "5                           ['jakarta']                 1.0       True    4.0   \n",
+       "13                         ['virginia']                 1.0       True    9.0   \n",
+       "...                                 ...                 ...        ...    ...   \n",
+       "5699                      ['cartagena']                 1.0      False   11.0   \n",
+       "5700                    ['los angeles']                 1.0       True    1.0   \n",
+       "5701                         ['durban']                 1.0       True   11.0   \n",
+       "5702                      ['melbourne']                 1.0      False   10.0   \n",
+       "5705                       ['new york']                 1.0      False   12.0   \n",
+       "\n",
+       "      Week                                   Headline_Details  \\\n",
+       "0     21.0  Grasberg Mine- Grasberg mine workers extend st...   \n",
+       "2     17.0  Shanghai port congestion impacts terminals in ...   \n",
+       "4     21.0  UPDATE - Indonesia: Police confirm two explosi...   \n",
+       "5     16.0  UPDATE - Indonesia: Severe winds damage infras...   \n",
+       "13    38.0  2 miles E of Chesterfield - A tornado has touc...   \n",
+       "...    ...                                                ...   \n",
+       "5699  48.0  Wildfire started by lightning strike near Cart...   \n",
+       "5700   2.0  Wind Event to Impact Southern CA This Week; Da...   \n",
+       "5701  46.0  Wind stoppages expected for the Port of Durban...   \n",
+       "5702  42.0  Wind warnings issue for Melbourne coast for Oc...   \n",
+       "5705  51.0  Winter storm may bring inclement weather condi...   \n",
+       "\n",
+       "                                                    url  \\\n",
+       "0     https://news.google.com/rss/articles/CBMiZ2h0d...   \n",
+       "2     https://news.google.com/rss/articles/CBMiVWh0d...   \n",
+       "4     https://news.google.com/rss/articles/CBMiZWh0d...   \n",
+       "5     https://news.google.com/rss/articles/CBMiSWh0d...   \n",
+       "13    https://news.google.com/rss/articles/CBMigAFod...   \n",
+       "...                                                 ...   \n",
+       "5699  https://news.google.com/rss/articles/CBMibmh0d...   \n",
+       "5700  https://news.google.com/rss/articles/CBMihgFod...   \n",
+       "5701  https://news.google.com/rss/articles/CBMiigFod...   \n",
+       "5702  https://news.google.com/rss/articles/CBMieGh0d...   \n",
+       "5705  https://news.google.com/rss/articles/CBMia2h0d...   \n",
+       "\n",
+       "                                                  title  \\\n",
+       "0     Freeport Indonesia mine workers extend strike ...   \n",
+       "2     Typhoon Muifa to shut China ports for second t...   \n",
+       "4     Jakarta Police Receive 2 More Reports on Coldp...   \n",
+       "5     Indonesia hit by some of strongest winds recorded   \n",
+       "13    UPDATE: Number of homes without power down to ...   \n",
+       "...                                                 ...   \n",
+       "5699  Neanderthals Were As Smart As Modern Humans an...   \n",
+       "5700  Hurricane Hilary live updates: Newsom declares...   \n",
+       "5701  Severe thunderstorms expected in most parts of...   \n",
+       "5702  NRL Draw 2023: All the scores, results and sch...   \n",
+       "5705  Big storm to dump heavy rain and snow along Ea...   \n",
+       "\n",
+       "                                                content  \n",
+       "0     Trucks are seen on a road in the Grasberg copp...  \n",
+       "2     By Sam Whelan 13/09/2022\\n\\nAnother typhoon ha...  \n",
+       "4     TEMPO.CO, Jakarta - South Jakarta Metro Police...  \n",
+       "5     A man stands near damaged houses following a t...  \n",
+       "13    More than 90,000 homes and businesses across t...  \n",
+       "...                                                 ...  \n",
+       "5699  Representational image (gorodenkoff, Getty Ima...  \n",
+       "5700  Staff reports\\n\\nPalm Springs Desert Sun\\n\\nLo...  \n",
+       "5701  DURBAN - Severe thunderstorms are expected in ...  \n",
+       "5702  The 2023 NRL season has finally come to an end...  \n",
+       "5705  A sprawling winter storm known as a nor’easter...  \n",
+       "\n",
+       "[2786 rows x 20 columns]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "not_found = [\"cannot scrape the url\", \"no url found\", None]\n",
+    "df2 = df[~df[\"url\"].isin(not_found)].drop_duplicates(\"url\")\n",
+    "df2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "465acb47-5f57-4d90-a272-7972d56e002b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(2786, 20)"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df2.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "63e26617",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import evaluate\n",
+    "\n",
+    "bert_score = evaluate.load(\"bertscore\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "id": "c824b8ce",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "\n",
+    "def compute_bert_score(predictions, references, max_length=510, debug=False):\n",
+    "    if debug:\n",
+    "        print(\"predictions:\", predictions)\n",
+    "        print(\"references:\", references)\n",
+    "\n",
+    "    predictions = [p[:max_length] if isinstance(p, str) else \"\" for p in predictions]\n",
+    "    references = [r[:max_length] for r in references]\n",
+    "\n",
+    "    bert_scores = bert_score.compute(\n",
+    "        predictions=predictions,\n",
+    "        references=references,\n",
+    "        lang=\"en\",\n",
+    "        model_type=\"microsoft/deberta-large-mnli\",\n",
+    "        rescale_with_baseline=True,\n",
+    "    )\n",
+    "    if debug:\n",
+    "        print(\"bert_scores:\", bert_scores)\n",
+    "\n",
+    "    for f1 in bert_scores[\"f1\"]:\n",
+    "        if f1 <= 0:\n",
+    "            return 0\n",
+    "    return np.mean(bert_scores[\"f1\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "d964d7e7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "predictions: ['Freeport Indonesia mine workers extend strike for fourth month', \"Trucks are seen on a road in the Grasberg copper and gold mine operated by an Indonesian subsidiary of Freeport-McMoRan Inc, near Timika, Papua province Thomson Reuters\\n\\nBy Wilda Asmarini and Susan Taylor\\n\\nAdvertisement\\n\\nJAKARTA/TORONTO (Reuters) - An estimated 5,000 workers at the giant Grasberg copper mine operated by Freeport-McMoRan Inc's Indonesian unit will extend their strike for a fourth month, a union official said on Friday, in an ongoing dispute over layoffs and employment terms.\\n\\nThe escalating labor issue comes as Freeport, the world's largest publicly traded copper miner, is snarled in a lengthy and costly dispute with Indonesia's government over rights to the Grasberg copper and gold mine.\\n\\nFreeport resumed copper concentrate exports from Grasberg, the world's second-largest copper mine, in April after a 15-week outage related to that row, but a permanent solution is yet to be found.\\n\\nCopper prices hit a 4-1/2 month peak on Friday, fueled by strong growth in top consumer China, a weak dollar and worries about supply disruptions.\\n\\nAdvertisement\\n\\nFreeport is pushing back against revised government rules that require miners to pay new taxes and royalties, divest a 51-percent stake and relinquish arbitration rights. The Arizona-based miner wants an 'investment stability agreement' that replicates the legal and fiscal rights under its existing agreement.\\n\\nFreeport Indonesia union industrial relations officer Tri Puspital told Reuters on Friday that the strike was extended because there is still no solution for worker concerns.\\n\\nThe strike began in May after Freeport laid off some 10 percent of its workforce to cut costs.\\n\\nIn May, Freeport said that mining and milling rates at Grasberg were affected by the strike, and investors will look for more information when the company reports second-quarter financial results July 25.\\n\\nAdvertisement\\n\\nIndonesia said last week it would invite Freeport chief executive Richard Adkerson to Jakarta this month to try to settle a dispute, but a company spokesman would not confirm whether he would attend.\\n\\nFreeport shares were down about 1 percent on New York at $12.93 Friday morning.\\n\\n(Reporting by Wilda Asmarini in Jakarta, Susan Taylor in Toronto and Maytaal Angel in London; Editing by Andrea Ricci)\"]\n",
+      "references: ['Grasberg Mine- Grasberg mine workers extend strike for a fourth month.', \"Media sources indicate that workers at the Grasberg mine will extend their strike for a fourth month as disputes regarding layoffs and benefits remains unresolved. Company officials report that operations at Grasberg Mine, located in Tembagapura, are running as normal despite to an ongoing strike by at least 6,000 mine workers. Around 25,000 workers and contractors are reported to remain on the job. The strike has been called over a dispute regarding employment conditions and layoffs of around 3,200 workers. The Grasberg Mine, primarily owned by Freeport McMoRan Inc, is the largest gold mine and third largest copper mine in the world; however, media sources indicate that the labor action has cut the mine's annual output by half. Freeport McMoRan has deemed the work stoppage illegal and has stated that workers absent for more than five days are considered to have resigned.\"]\n",
+      "bert_scores: {'precision': [0.6296835541725159, 0.1178620457649231], 'recall': [0.5122898817062378, 0.30864831805229187], 'f1': [0.5700059533119202, 0.20724298059940338], 'hashcode': 'microsoft/deberta-large-mnli_L18_no-idf_version=0.3.12(hug_trans=4.40.2)-rescaled'}\n",
+      "CPU times: total: 1.95 s\n",
+      "Wall time: 5.41 s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0.3886244669556618"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "index = 0\n",
+    "compute_bert_score(\n",
+    "    [df2[\"title\"].iloc[index], df2[\"content\"].iloc[index]],\n",
+    "    [df2[\"Headline\"].iloc[index], df2[\"Details\"].iloc[index]],\n",
+    "    debug=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "c65253ea",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "predictions: ['Typhoon Muifa to shut China ports for second time in 10 days', 'By Sam Whelan 13/09/2022\\n\\nAnother typhoon has prompted the ports of Shanghai and Ningbo to close for the second time in 10 days, with forwarders expecting a “ripple effect” of shipping delays.\\n\\nAlthough weaker than Typhoon Hinnamor – which saw Shanghai, Ningbo and Busan all suspend operations last week – Typhoon Muifa is on a direct path to hit Shanghai tomorrow, according to the US Joint Typhoon Warning Center.\\n\\nNorman Global Logistics (NGL) told customers Shanghai’s Yangshan container terminals were due to stop gate-in handling by 7pm local time today, while both Waigaoqiao and Yangshan terminals would be fully closed by 8am tomorrow morning.\\n\\nNingbo’s container terminals and yards were already closed this morning, NGL said.\\n\\nA spokeswoman for Dimerco said Muifa was due to pass Shanghai in “no more than a day”, but added that inland trucking would be “dangerous” during the typhoon’s impact.\\n\\n“Fleet schedules have been in a mess for a long time,” she added. “We don’t expect any remarkable impact, plus the market is super slow.”\\n\\nHowever, CH Robinson said ships were already waiting two-to-four days to berth at Shanghai and Ningbo due to Typhoon Hinnamor, and noted the impact from Muifa “could be similar.”\\n\\nLikewise, Shanghai-based Thomas Gronen, head of Greater China at Fibs Logistics, told The Loadstar “this one could have a bigger impact”.\\n\\nHe added: “We can expect at least two days’ impact on port operations, as the typhoon isn’t moving too fast, and then there will be an additional ripple effect towards the upcoming weekend departures.”\\n\\nShanghai and Ningbo are already dealing with increased berth congestion, too, according to Linerlytica.\\n\\nIt said: “Typhoon Hinnamnor is taking some time to clear out with the ports of Shanghai, Ningbo and Busan most badly affected of the North Asia ports.\\n\\n“Ships that were diverted to the Qingdao anchorage in Bohai to avoid the path of typhoon Hinnamnor last week moved back to their scheduled windows at Shanghai and Ningbo, driving congestion at the two main eastern China ports to a record high of 885,000 teu [at anchorage] last Thursday.\\n\\n“Although the queue of vessels has dropped to 500,000 teu by the end of the week, it will take another week before the ports work through the backlog of ships.”']\n",
+      "references: ['Shanghai port congestion impacts terminals in Qingdao and Ningbo', 'The persisting port congestion at Shanghai’s Yangshan deep-sea terminal is reportedly also impacting vessel arrival times at other major public terminals in China, including Qingdao and Ningbo. Industry sources indicated that this is due to some vessels being diverted from Shanghai as well as unstable weather and re-shuffling of alliances networks. Some carriers are reportedly planning contingency actions for the coming months, including consolidation of volumes and temporary reduction of port calls.  No further details were initially provided.']\n",
+      "bert_scores: {'precision': [-0.08345548808574677, -0.004735329654067755], 'recall': [0.04282763972878456, 0.17711813747882843], 'f1': [-0.021741997450590134, 0.080312080681324], 'hashcode': 'microsoft/deberta-large-mnli_L18_no-idf_version=0.3.12(hug_trans=4.40.2)-rescaled'}\n",
+      "CPU times: total: 1.03 s\n",
+      "Wall time: 6.9 s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "index = 1\n",
+    "compute_bert_score(\n",
+    "    [df2[\"title\"].iloc[index], df2[\"content\"].iloc[index]],\n",
+    "    [df2[\"Headline\"].iloc[index], df2[\"Details\"].iloc[index]],\n",
+    "    debug=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "a6eda1e8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "predictions: ['Indonesia hit by some of strongest winds recorded', 'A man stands near damaged houses following a tornado at Sukadana village in Sumedang, West Java province\\n\\nIndonesia has been struck by some of the strongest winds recorded in the country, injuring at least 33 people.\\n\\nBuildings were damaged as the winds tore through the town of Sumedang in West Java province.\\n\\nVideos uploaded to social media show debris flying through the air, roofs torn from buildings and part of a convenience store collapsing.\\n\\nA climatologist at government research body BRIN said winds were strong enough to be considered a tornado.\\n\\n\"My father was sitting in our terrace and suddenly he saw plastic just flying and twisting. Not long after that, the strong wind came near my house,\" Kay Tiara, a Sumedang resident, told Reuters.\\n\\n\"The roof of my house immediately flew away. My family and I took cover inside my house,\" she added.\\n\\nIndonesia\\'s disaster mitigation agency said no serious injuries have been reported.\\n\\nErma Yulihastin, a climatologist at BRIN, told BBC Indonesia that she believes the 65km/h (40mph) threshold had been met for it to be considered a tornado.\\n\\nHowever, Indonesia\\'s Meteorology Climatology and Geophysics Council (BMKG) stopped short of calling it such.\\n\\nThe winds lasted up to 20 minutes, Ms Yulihastin said. The previous strongest winds in Indonesia were seen in Cimenyan, Bandung, in 2021, when speeds of 56km/h (35mph) were recorded.']\n",
+      "references: ['UPDATE - Indonesia: Severe winds damage infrastructure in Bandung, West Java', 'Severe winds have downed billboards and trees in Bandung on Wednesday afternoon. According to local media sources, the gales were accompanied by heavy rain which caused localised flooding in certain areas. Available images show a large billboard collapsed against the Flyover Rainbow Antapani causing significant congestion on Jalan Terusan Jakarta. There are no reports of casualties at this time. Travellers in Bandung should drive with caution and be prepared to alter routes at short notice. Keep abreast of the latest weather forecasts and defer journeys if necessary.']\n",
+      "bert_scores: {'precision': [0.2372419238090515, 0.03713945671916008], 'recall': [0.19886218011379242, 0.15693290531635284], 'f1': [0.22037585079669952, 0.09625885635614395], 'hashcode': 'microsoft/deberta-large-mnli_L18_no-idf_version=0.3.12(hug_trans=4.40.2)-rescaled'}\n",
+      "CPU times: total: 172 ms\n",
+      "Wall time: 566 ms\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0.15831735357642174"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "index = 3\n",
+    "compute_bert_score(\n",
+    "    [df2[\"title\"].iloc[index], df2[\"content\"].iloc[index]],\n",
+    "    [df2[\"Headline\"].iloc[index], df2[\"Details\"].iloc[index]],\n",
+    "    debug=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "f6bad3fb",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "predictions: ['Cops open fire on hostage-taking kidnappers after 4-hour standoff in victim’s semi-truck: video', 'A multi-county police chase turned into a 4-hour hostage standoff in the middle of an Ohio highway before cops opened fire on the two suspects, who later died Wednesday morning.\\n\\nHarrowing footage showed several Ohio Highway Patrol Troopers in tactical gear approach the hijacked semi-truck in an attempt to free the driver, who was believed to have been held hostage inside by a 54-year-old man and his 51-year-old wife.\\n\\nOne of the suspects began shooting out of the truck and was quickly met with return fire that seemingly struck both suspects.\\n\\nOne trooper can be seen unloading over 20 rounds from his pistol as the group backs away from the site of the shooter inside, according to the video obtained by WBNS 10TV.\\n\\nWhen the shooting ended, the original driver of the truck escaped the vehicle with his hands up and was brought to safety by two troopers.\\n\\nThe two suspects, who said their names were Ronald and Barbara Taylor of Somerset, Ky., were transported to the Miami Valley Hospital in Dayton, where both died of their injuries.\\n\\nThe chaotic morning began just before 1 a.m. on Wednesday when a London (Ohio) police officer stopped a minivan with no rear lights or a visible registration.\\n\\n7 Harrowing footage showed several Ohio Highway Patrol Troopers in tactical gear approach the hijacked semi-truck in an attempt to free the driver who was believed to have been held hostage. WBNS 10TV/YouTube\\n\\n7 One trooper can be seen unloading over 20 rounds from his pistol as the group backs away from the site of the shooter inside. WBNS 10TV/YouTube\\n\\nIn body cam footage provided by the London police, the officer tried to get the identities of the couple but both said they didn’t have ID on them, adding that the woman had just left the emergency room after stabbing herself in the eye with scissors.\\n\\nThe male driver then explained to the officer they don’t have any form of ID because it was all in the woman’s purse that was stolen.\\n\\nEventually, they identified themselves as Ronald and Barbara Taylor before the male driver took off, leading officers on a 4-minute chase down to a truck stop, where they fled the minivan, and a foot chase ensued around the parking lot.\\n\\nAs the initial officer caught up to the couple, he threatened to taser them, but the male suspect pointed a handgun back forcing the cop to dive behind a parked car and allowing the couple to flee to a nearby white Werner semi-tractor that had one male occupant inside.\\n\\n7 Eventually, they identified themselves as Ronald and Barbara Taylor before the male driver took off, leading officers on a 4-minute chase down to a truck stop, where they fled the minivan and a foot chase pursued around through the parking lot. London (OH) Police\\n\\n7 It the couple took over 20 minutes to put the truck in drive, police set up a barricade of patrol cars and spike strips to stop the couple from getting out of the parking lot. London (OH) Police\\n\\n7 The couple fled and was chased for 2 hours through several counties before stopping near Dayton International Airport. London (OH) Police\\n\\nThe couple took over 20 minutes to put the truck in drive, which gave the police time to set up a barricade of patrol cars and spike strips to stop the couple from getting out of the parking lot.\\n\\nHowever, the plan backfired as the truck drove straight into one of the police cars.\\n\\nThe chase lasted for two hours, going through several counties including Clark, Greene, Madison, and Montgomery.\\n\\nIt was in Montgomery when the truck rolled to a stop on the Dayton International Airport Access Road, according to WHIO TV.\\n\\nBetween 3:20 and 7:30 a.m. officers from multiple agencies tried negotiating with the couple to release their hostage, while the access road was shut down.\\n\\n“There were negotiations that were occurring from around 3:20 a.m., when the vehicle came to a stop, till about 7:30 this morning when troopers from the Special Response Team decided to attempt to make entry.” Ohio State Highway Patrol Lieutenant Natan Dennis said.\\n\\n7 The aftermath of the shootout shows several bullet holes in the truck’s windows. WBNS 10TV/YouTube\\n\\n7 The truck was left with multiple bullet holes from the shooting, and without a left front tire after it ran over a spike strip during chase, leaving only the rim. WBNS 10TV/YouTube\\n\\n“Troopers from the special response team approached the truck to remove the victim from the truck, as they were making entry the suspects fired shots at troopers,” Dennis added. “Troopers returned fire on the suspects.”\\n\\nNumerous bullet holes were seen on the truck’s windshield after the incident.']\n",
+      "references: ['6 suspects steal truck and take driver hostage in Santos', 'Local media sources indicate on October 18 that 6 suspects hijacked a truck in Santos on the evening of October 15, releasing the driver in Sao Bernardo do Campo. The truck was carrying BRL 35,000 (EUR 8232.62; USD 9432.15) in potassium chlorate. 4 of the 6 suspects have been apprehended so far, the cargo is still missing, and the remaining 2 suspects are still at large. Distributors to and from the Port of Santos and the ABC manufacturing region should be mindful of cargo theft risks and should plan accordingly.']\n",
+      "bert_scores: {'precision': [0.18912100791931152, -0.14989086985588074], 'recall': [0.2307809293270111, 0.02283167839050293], 'f1': [0.2122330665588379, -0.06960325688123703], 'hashcode': 'microsoft/deberta-large-mnli_L18_no-idf_version=0.3.12(hug_trans=4.40.2)-rescaled'}\n",
+      "CPU times: total: 7.36 s\n",
+      "Wall time: 18.6 s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "index = 10\n",
+    "compute_bert_score(\n",
+    "    [df2[\"title\"].iloc[index], df2[\"content\"].iloc[index]],\n",
+    "    [df2[\"Headline\"].iloc[index], df2[\"Details\"].iloc[index]],\n",
+    "    debug=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "bb0c31bb",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "predictions: ['Typhoon Doksuri heads for China after battering Philippines', 'At least 20 are killed after boat capsizes in Philippines; China, Taiwan issue severe weather alerts.\\n\\nChina has braced for the imminent landfall of Typhoon Doksuri, as its national observatory renewed its most severe weather alert after overnight heavy rainfall in the country’s southwest.\\n\\nAccording to a state media report, three coastal cities in Fujian province shut schools, businesses and factories on Thursday, while flood control authorities in one of them, Xiamen, warned of a “serious impact”.\\n\\nIn the Philippines, at least eight people died as the typhoon savaged the region with strong winds and heavy rains.\\n\\nOn Thursday, more than 20 people were confirmed dead in the country after a passenger boat capsized.\\n\\nThe MBCA Princess Aya was just 45 metres from the shore of Binangonan town in Rizal province, east of Manila, when it was pummelled on its way to Talim Island by a gust of wind, causing the passengers to panic.\\n\\n“They went to the port side of the motor banca, causing the boat to tilt and capsize,” coastguard spokesman Rear Admiral Armand Balilo told a Manila radio station.\\n\\nTyphoon Doksuri battered the main island of Luzon on Wednesday, toppling trees, knocking out power and forcing the evacuation of thousands of people from coastal communities.\\n\\nStrong winds and heavy rain pummelled the lightly populated Babuyan islands and northern provinces, triggering flooding and landslides.\\n\\nTyphoon threatens China\\n\\nThe approaching typhoon is expected to make landfall on China’s southeast coast in the early hours of Friday.\\n\\nFifteen provinces and city-level administrative units across China have been affected by “severe” weather, including thunderstorms, heavy rainfall, gales and hail ahead of Doksuri’s landfall, state news agency Xinhua reported.\\n\\nBeijing launched emergency flood control operations in the country’s southwest on Wednesday night after torrential rains in the Sichuan, Guizhou and Yunnan provinces as well as the nearby metropolis of Chongqing.\\n\\nHeavy flooding in the city of Luzhou, Sichuan province, swept cars onto tree trunks, according to videos circulating on Chinese social media.\\n\\nPassenger ships and fishing boats have also been grounded in parts of coastal Zhejiang province north of Fujian.\\n\\nChina’s national observatory has classified Doksuri as a “strong” typhoon, with maximum winds of 180km/h (112mph), as it hurtled northwest through the Taiwan Strait towards Fujian province as of 12pm (04:00 GMT) on Thursday.\\n\\nAt one point Doksuri was a super typhoon, but lost some of its strength after it lashed the coastline of the northern Philippines on Wednesday.\\n\\nTaiwan warns of landslides\\n\\nSouthern Taiwan shut businesses and schools, and hundreds of flights were cancelled amid warnings of landslides and floods on Thursday.\\n\\nTaiwan’s weather bureau issued wind and rain warnings for the southern and eastern parts of the island, including the major port city of Kaohsiung where landslide warnings were issued.\\n\\nRailway services between southern and eastern Taiwan were shut.\\n\\nMore than 5,700 people were evacuated as a precaution, mostly in mountainous southern and eastern Taiwan, where more than 700mm of rainfall was recorded in some areas and up to 1,000mm of rain was forecast.\\n\\nThe storm had cut power from more than 49,000 households across Taiwan but the majority of them had since been restored.']\n",
+      "references: ['7 ships grounded in Taiwan due to Tropical Depression', 'Media sources reported that 6 container ships and a tanker were grounded due to the passing of a Tropical Depression near Taiwan on August 24. In a series of maritime accident, Container ship Jin Hua was grounded at 05:00 (local time) at north of Kaohsiung in Tainan city area. Simultaneously, container ship and a product tanker, Unlimited 2 and Dragon Aria were grounded at 05:00 somewhere between Kaohsiung and Tainan as well as at Kaohsiung Port’s entrance respectively. Other cargo ships, Tai Cang Hu, An Li 669, Shun Hong, Chang Long 68 were also grounded on the same morning. According to sources, emergency meeting is in place to handle the issue.']\n",
+      "bert_scores: {'precision': [0.14044061303138733, 0.020360536873340607], 'recall': [0.1391707956790924, 0.22829866409301758], 'f1': [0.1427447646856308, 0.11602437496185303], 'hashcode': 'microsoft/deberta-large-mnli_L18_no-idf_version=0.3.12(hug_trans=4.40.2)-rescaled'}\n",
+      "CPU times: total: 3.12 s\n",
+      "Wall time: 11 s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0.1293845698237419"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "index = 11\n",
+    "compute_bert_score(\n",
+    "    [df2[\"title\"].iloc[index], df2[\"content\"].iloc[index]],\n",
+    "    [df2[\"Headline\"].iloc[index], df2[\"Details\"].iloc[index]],\n",
+    "    debug=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "55991d37",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "predictions: ['21 km NNE of Hualien City, Taiwan, on Saturday, Apr 27, 2024, at 02:21 am (Taipei Time)', 'Strong magnitude 5.7 earthquake at 33 km depth\\n\\n26 Apr 18:25 UTC: First to report: VolcanoDiscovery after 4 minutes.\\n\\n26 Apr 18:27: Now using data updates from EMSC\\n\\n... [show all] ... 26 Apr 18:28: Epicenter location corrected by 4.9 km (3 mi) towards N.\\n\\n26 Apr 18:30: Hypocenter depth recalculated from 20.0 to 30.7 km (from 12.4 to 19.1 mi). Epicenter location corrected by 2.2 km (1.4 mi) towards E.\\n\\n26 Apr 18:31: Magnitude recalculated from 5.8 to 5.7. Hypocenter depth recalculated from 30.7 to 31.4 km']\n",
+      "references: ['91 miles SE of Taipei - A magnitude 5.7 earthquake struck in the Philippine Sea. Incident closed.', 'The United States Geological Survey (USGS) reported a magnitude 5.7 earthquake that struck in the Philippine Sea, approximately 91 miles southeast of Taipei, Taiwan. There were no reports of associated injuries or structural damage. The earthquake, which was originally recorded with a magnitude of 6.0, occurred at a depth of approximately 21 miles. This incident is closed.']\n",
+      "bert_scores: {'precision': [0.11710108071565628, -0.07145067304372787], 'recall': [0.14049701392650604, 0.054687317460775375], 'f1': [0.13163256645202637, -0.009768012911081314], 'hashcode': 'microsoft/deberta-large-mnli_L18_no-idf_version=0.3.12(hug_trans=4.40.2)-rescaled'}\n",
+      "CPU times: total: 125 ms\n",
+      "Wall time: 2.13 s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "index = 12\n",
+    "compute_bert_score(\n",
+    "    [df2[\"title\"].iloc[index], df2[\"content\"].iloc[index]],\n",
+    "    [df2[\"Headline\"].iloc[index], df2[\"Details\"].iloc[index]],\n",
+    "    debug=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "id": "3ab2c7db",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "predictions: ['Worst fires and explosions in Vietnam in 2022', nan]\n",
+      "references: ['Large fire destroys a fabric warehouse in Ho Chi Minh city', 'Vietnamese media sources reported that a large fire broke out at a warehouse that stores fabric in Vuon Lai, An Phu Dong, 12 district of Ho Chi Minh City. According to sources, the fire broke out at 13:00 (local time) and it quickly spread to the warehouse area where it stored assets and materials. Reports suggested that the entire warehouse was destroyed. No immediate reports of casualties. The cause of the incident is likely to be due to a fault of an electrical short circuit.']\n",
+      "bert_scores: {'precision': [0.1329418420791626, -0.8301020860671997], 'recall': [0.16975580155849457, -0.8302450776100159], 'f1': [0.15390285849571228, -0.8239193558692932], 'hashcode': 'microsoft/deberta-large-mnli_L18_no-idf_version=0.3.12(hug_trans=4.40.2)-rescaled'}\n",
+      "CPU times: total: 46.9 ms\n",
+      "Wall time: 2 s\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Warning: Empty candidate sentence detected; setting raw BERTscores to 0.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0"
+      ]
+     },
+     "execution_count": 44,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "index = 214\n",
+    "compute_bert_score(\n",
+    "    [df2[\"title\"].iloc[index], df2[\"content\"].iloc[index]],\n",
+    "    [df2[\"Headline\"].iloc[index], df2[\"Details\"].iloc[index]],\n",
+    "    debug=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "b4ff3568",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tqdm.notebook import tqdm\n",
+    "\n",
+    "tqdm.pandas()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "id": "16d795e2",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "68ccd8cb48c74bf1b2774efb3e79b4b1",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/2786 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Warning: Empty candidate sentence detected; setting raw BERTscores to 0.\n",
+      "Warning: Empty candidate sentence detected; setting raw BERTscores to 0.\n",
+      "Warning: Empty candidate sentence detected; setting raw BERTscores to 0.\n",
+      "Warning: Empty candidate sentence detected; setting raw BERTscores to 0.\n",
+      "Warning: Empty candidate sentence detected; setting raw BERTscores to 0.\n",
+      "Warning: Empty candidate sentence detected; setting raw BERTscores to 0.\n",
+      "Warning: Empty candidate sentence detected; setting raw BERTscores to 0.\n",
+      "Warning: Empty candidate sentence detected; setting raw BERTscores to 0.\n",
+      "Warning: Empty candidate sentence detected; setting raw BERTscores to 0.\n",
+      "Warning: Empty candidate sentence detected; setting raw BERTscores to 0.\n",
+      "Warning: Empty candidate sentence detected; setting raw BERTscores to 0.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: total: 4min 17s\n",
+      "Wall time: 15min 7s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "df2[\"bert_score\"] = df2.progress_apply(\n",
+    "    lambda x: compute_bert_score(\n",
+    "        [x[\"title\"], x[\"content\"]], [x[\"Headline\"], x[\"Details\"]]\n",
+    "    ),\n",
+    "    axis=1,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 88,
+   "id": "d55618d5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(15, 21)"
+      ]
+     },
+     "execution_count": 88,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df3 = df2[df2[\"bert_score\"] >= 0.5]\n",
+    "df3.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 90,
+   "id": "ee72fadf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def print_row(row):\n",
+    "    print(\"title:\", row[\"title\"])\n",
+    "    print(\"=\" * 100)\n",
+    "    print(\"content:\", row[\"content\"][:510])\n",
+    "    print(\"=\" * 100)\n",
+    "    print(\"Headline:\", row[\"Headline\"])\n",
+    "    print(\"=\" * 100)\n",
+    "    print(\"Details:\", row[\"Details\"][:510])\n",
+    "    print(\"=\" * 100)\n",
+    "    print(\"bert_score:\", row[\"bert_score\"])\n",
+    "    print(\"=\" * 100)\n",
+    "    print(\"url:\", row[\"url\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 91,
+   "id": "544ce457",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "title: Iranian oil tanker damaged by explosions near Saudi port city\n",
+      "====================================================================================================\n",
+      "content: Two missiles hit an Iranian state-owned oil tanker as it headed to Syria on the Red Sea, the Iranian government has claimed.\n",
+      "\n",
+      "Tehran did not attribute responsibility immediately but said two explosions 20 minutes apart on the Sabiti tanker, which caused oil to spill from two tanks, were not the result of an accident.\n",
+      "\n",
+      "The boat was about 60 miles from the Saudi Arabian port city of Jeddah when it was hit. The scale of the damage did not appear extensive in photos published by Tehran news agencies.\n",
+      "\n",
+      "The bla\n",
+      "====================================================================================================\n",
+      "Headline: Iranian oil tanker damaged by explosions near Port of Jeddah\n",
+      "====================================================================================================\n",
+      "Details: On October 11, media sources indicate that an oil tanker belonging to the National Iranian Oil Company (NIOC) was damaged by explosions while passing Port of Jeddah, Saudi Arabia in the Red Sea, on Friday, October 11. Explosions, likely caused by missile strikes, resulted in the crude oil to leak into the sea. According to IRNA, the tanker was about 60 miles (96 km) from the Saudi port of Jeddah when the incident occurred. All members of the tanker's crew are reportedly safe. Oil prices increased by 2% af\n",
+      "====================================================================================================\n",
+      "bert_score: 0.5100782811641693\n",
+      "====================================================================================================\n",
+      "url: https://news.google.com/rss/articles/CBMicGh0dHBzOi8vd3d3LnRoZWd1YXJkaWFuLmNvbS93b3JsZC8yMDE5L29jdC8xMS9pcmFuaWFuLW9pbC10YW5rZXItb24tZmlyZS1hZnRlci1ibGFzdC1uZWFyLXNhdWRpLXBvcnQtY2l0eS1yZXBvcnTSAXBodHRwczovL2FtcC50aGVndWFyZGlhbi5jb20vd29ybGQvMjAxOS9vY3QvMTEvaXJhbmlhbi1vaWwtdGFua2VyLW9uLWZpcmUtYWZ0ZXItYmxhc3QtbmVhci1zYXVkaS1wb3J0LWNpdHktcmVwb3J0?oc=5&hl=en-SG&gl=SG&ceid=SG:en\n"
+     ]
+    }
+   ],
+   "source": [
+    "row = df3.loc[df3[\"bert_score\"].idxmin()]\n",
+    "\n",
+    "print_row(row)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 76,
+   "id": "7735e024",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from googlenewsdecoder import new_decoderv1\n",
+    "\n",
+    "\n",
+    "def find_true_url(url, debug=False):\n",
+    "    if \"news.google.com\" not in url:\n",
+    "        return url\n",
+    "\n",
+    "    if debug:\n",
+    "        print(\"finding true url for:\", url)\n",
+    "    interval_time = 5  # default interval is 1 sec, if not specified\n",
+    "\n",
+    "    try:\n",
+    "        decoded_url = new_decoderv1(url, interval=interval_time)\n",
+    "        if debug:\n",
+    "            print(\"decoded_url:\", decoded_url)\n",
+    "\n",
+    "        if decoded_url.get(\"status\"):\n",
+    "            return decoded_url[\"decoded_url\"]\n",
+    "        else:\n",
+    "            print(\"Error:\", decoded_url[\"message\"])\n",
+    "    except Exception as e:\n",
+    "        print(f\"Error occurred: {e}\")\n",
+    "\n",
+    "    return None"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 93,
+   "id": "064686a6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "finding true url for: https://news.google.com/rss/articles/CBMiWGh0dHBzOi8vc3BsYXNoMjQ3LmNvbS9qYXBhbmVzZS11bHRyYS1sYXJnZS1jb250YWluZXJzaGlwLXNsYW1zLWludG8tYnVzYW4tZ2FudHJ5LWNyYW5lcy_SAQA?oc=5&hl=en-SG&gl=SG&ceid=SG:en\n",
+      "decoded_url: {'status': True, 'decoded_url': 'https://splash247.com/japanese-ultra-large-containership-slams-into-busan-gantry-cranes/'}\n",
+      "CPU times: total: 15.6 ms\n",
+      "Wall time: 26.2 s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'https://splash247.com/japanese-ultra-large-containership-slams-into-busan-gantry-cranes/'"
+      ]
+     },
+     "execution_count": 93,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "find_true_url(df3.url.iloc[-1], debug=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 92,
+   "id": "a781c176",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "title: Japanese mega boxship slams into cranes at Busan New Port\n",
+      "====================================================================================================\n",
+      "content: Yet another gantry crane has been pulverised by a poorly berthed mega boxship. The Japanese controlled 13,900 teu Milano Bridge , part of the Ocean Network Express (ONE) fleet, smashed into the quayside at Busan New Port on Monday, bringing down a crane onto the back end of the boxship in the process.\n",
+      "\n",
+      "Dramatic footage sent to Splash shows the incident, which also sees the vessel come into contact with 10,000 teu boxship Seaspan Ganges.\n",
+      "\n",
+      "As well as bringing down crane 85, at least two other cranes sustain\n",
+      "====================================================================================================\n",
+      "Headline: Japanese mega boxship slams into cranes at Busan New Port\n",
+      "====================================================================================================\n",
+      "Details: Media sources indicated on April 7 that Japanese mega boxship Milano Bridge smashed into the quayside at Busan New Port on April 6, causing a crane onto the back end of the boxship in the process. Milano Bridge is part of the Ocean Network Express fleet.\n",
+      "====================================================================================================\n",
+      "bert_score: 0.6692844182252884\n",
+      "====================================================================================================\n",
+      "url: https://news.google.com/rss/articles/CBMiWGh0dHBzOi8vc3BsYXNoMjQ3LmNvbS9qYXBhbmVzZS11bHRyYS1sYXJnZS1jb250YWluZXJzaGlwLXNsYW1zLWludG8tYnVzYW4tZ2FudHJ5LWNyYW5lcy_SAQA?oc=5&hl=en-SG&gl=SG&ceid=SG:en\n"
+     ]
+    }
+   ],
+   "source": [
+    "print_row(df3.iloc[-1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 94,
+   "id": "7030c7a3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e9266641b2f9447f8132adbcf929dcc4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/15 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: total: 46.9 ms\n",
+      "Wall time: 11min 48s\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "<timed exec>:1: SettingWithCopyWarning: \n",
+      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
+      "Try using .loc[row_indexer,col_indexer] = value instead\n",
+      "\n",
+      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "\n",
+    "df3[\"url\"] = df3[\"url\"].progress_apply(find_true_url)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 95,
+   "id": "172e775c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df3.to_csv(\"data/scrapped_data2_cleaned.csv\", index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 107,
+   "id": "84b1e56c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "*************************************************01*************************************************\n",
+      "title: 270 kg of heroin found in container at Port of Genoa\n",
+      "====================================================================================================\n",
+      "content: The greatest heroin seizure of the last twenty years was carried out at the Port of Genoa. 270 kilos of heroin was found in a container coming from Iran after a long investigation by the Italian police, assisted by the staff of the Customs Agency. The container had ultimate destination the Netherlands.\n",
+      "\n",
+      "According to local reports, once the drugs was seized, the Italian police organised and executed the first controlled delivery of heroin outside Italy. Under the coordination of law enforcement agencies an\n",
+      "====================================================================================================\n",
+      "Headline: 270 kilograms of heroin discovered in container at Port of Genoa\n",
+      "====================================================================================================\n",
+      "Details: Local media sources indicated on November 8 that 270 kilograms of heroin arrived from Iran were discovered in containers at the Port of Genoa by Italian Customs. Two suspects were arrested, as the illicit cargo was believed to be bound for Switzerland, France, Belgium, and the Netherlands. Distributors should be mindful of illicit activity risks to their maritime shipments and should plan accordingly.\n",
+      "====================================================================================================\n",
+      "bert_score: 0.6463567316532135\n",
+      "====================================================================================================\n",
+      "url: https://safety4sea.com/270-kg-of-heroin-found-in-container-at-port-of-genoa/\n",
+      "\n",
+      "*************************************************02*************************************************\n",
+      "title: Five bogies of freight train derail near Mirpur Mathelo\n",
+      "====================================================================================================\n",
+      "content: Five bogies of a freight train derailed on Monday morning near Mirpur Mathelo railway station in Sindh. The Karachi-bound train’s derailment caused suspension of railway traffic on the down track, officials said. The train was running to Karachi from Sahiwal to carry back coal cargo from the port city. Various trains running to Karachi were stopped at Daharki and Sadiqabad railway stations, sources said. The derailment incident has caused damage to the bogies and railway track, railway department sources \n",
+      "====================================================================================================\n",
+      "Headline: 5 freight cars derail near Mirpur Mathelo\n",
+      "====================================================================================================\n",
+      "Details: Local media sources indicate on October 14 that 5 cars of a Sahiwal to Karachi freight train derailed near Mirpur Mathelo on October 14, resulting in a suspension of Karachi-bound traffic. Due to the damage to cars and track, the line won't be restored until the evening of October 14. Those shipping by rail to Karachi should anticipate delays and should plan accordingly.\n",
+      "====================================================================================================\n",
+      "bert_score: 0.5540391951799393\n",
+      "====================================================================================================\n",
+      "url: https://dailytimes.com.pk/483777/five-bogies-of-freight-train-derail-near-mirpur-mathelo/\n",
+      "\n",
+      "*************************************************03*************************************************\n",
+      "title: Dutch customs seize 90,000 bottles of vodka believed to be for Kim Jong-un\n",
+      "====================================================================================================\n",
+      "content: Dutch customs officials at the port of Rotterdam have seized 90,000 bottles of vodka believed to be destined for the North Korean leader, Kim Jong-un, and his army chiefs.\n",
+      "\n",
+      "The discovery, on the eve of Kim’s two-day summit with Donald Trump in Hanoi, was made after Dutch authorities flagged up the suspicious route and records of a Chinese-owned container ship, Nebula.\n",
+      "\n",
+      "The Russian vodka, contained in 3,000 boxes, had been recorded as being due for unloading in China, via the ports of Hamburg and Rotterdam\n",
+      "====================================================================================================\n",
+      "Headline: Dutch customs seize 90,000 bottles of vodka to be exported to North Korea\n",
+      "====================================================================================================\n",
+      "Details: Local media sources reported on February 25, 2019 that Dutch customs officials at the port of Rotterdam have seized 90,000 bottles of vodka believed to be destined for the North Korean leader, Kim Jong-un, and his army chiefs. The discovery was made after Dutch authorities flagged up the suspicious route and records of a Chinese-owned container ship, Nebula. The Russian vodka, contained in 3,000 boxes, had been recorded as being due for unloading in China, via the ports of Hamburg and Rotterdam. When offi\n",
+      "====================================================================================================\n",
+      "bert_score: 0.7640108466148376\n",
+      "====================================================================================================\n",
+      "url: https://www.theguardian.com/world/2019/feb/26/dutch-customs-seize-90000-bottles-of-vodka-believed-to-be-for-kim-jong-un\n",
+      "\n",
+      "*************************************************04*************************************************\n",
+      "title: Iranian oil tanker damaged by explosions near Saudi port city\n",
+      "====================================================================================================\n",
+      "content: Two missiles hit an Iranian state-owned oil tanker as it headed to Syria on the Red Sea, the Iranian government has claimed.\n",
+      "\n",
+      "Tehran did not attribute responsibility immediately but said two explosions 20 minutes apart on the Sabiti tanker, which caused oil to spill from two tanks, were not the result of an accident.\n",
+      "\n",
+      "The boat was about 60 miles from the Saudi Arabian port city of Jeddah when it was hit. The scale of the damage did not appear extensive in photos published by Tehran news agencies.\n",
+      "\n",
+      "The bla\n",
+      "====================================================================================================\n",
+      "Headline: Iranian oil tanker damaged by explosions near Port of Jeddah\n",
+      "====================================================================================================\n",
+      "Details: On October 11, media sources indicate that an oil tanker belonging to the National Iranian Oil Company (NIOC) was damaged by explosions while passing Port of Jeddah, Saudi Arabia in the Red Sea, on Friday, October 11. Explosions, likely caused by missile strikes, resulted in the crude oil to leak into the sea. According to IRNA, the tanker was about 60 miles (96 km) from the Saudi port of Jeddah when the incident occurred. All members of the tanker's crew are reportedly safe. Oil prices increased by 2% af\n",
+      "====================================================================================================\n",
+      "bert_score: 0.5100782811641693\n",
+      "====================================================================================================\n",
+      "url: https://www.theguardian.com/world/2019/oct/11/iranian-oil-tanker-on-fire-after-blast-near-saudi-port-city-report\n",
+      "\n",
+      "*************************************************05*************************************************\n",
+      "title: Mexico intercepts 80kg of cocaine on CMA CGM ship\n",
+      "====================================================================================================\n",
+      "content: Mexican Navy and Coast Guard personnel have seized over 80 kilograms of cocaine from a container aboard the CMA CGM Mississippi at the Pacific Ocean Port of Lázaro Cárdenas.\n",
+      "\n",
+      "The navy said that a total of 70 packages containing approximately 82 kilograms of narcotics were seized in an operation on September 18.\n",
+      "\n",
+      "According to the cargo manifest, the container on the Post-Panamax ship was supposed to be filled with electrical cable rolls. However, after opening the container, the authorities found bags with\n",
+      "====================================================================================================\n",
+      "Headline: Mexican Navy seizes 80 kg of cocaine on CMA CGM container ship in the Port of Lazaro Cardenas\n",
+      "====================================================================================================\n",
+      "Details: On September 24, sources report that Mexican Navy and Coast Guard personnel seized over 80 kilograms of cocaine from a container aboard the CMA CGM Mississippi at the Pacific Ocean Port of Lazaro Cardenas. A total of 70 packages containing approximately 82 kilograms of narcotics were reportedly seized in an operation on September 18. An investigation into the matter has been launched. It was not immediately reported whether the vessel was detained for a longer period of time in the port. The containership\n",
+      "====================================================================================================\n",
+      "bert_score: 0.5610589683055878\n",
+      "====================================================================================================\n",
+      "url: https://shipsandports.com.ng/mexico-intercepts-80kg-of-cocaine-on-cma-cgm-ship-2/\n",
+      "\n",
+      "*************************************************06*************************************************\n",
+      "title: 15 people arrested for illegal MGO transaction in Singapore\n",
+      "====================================================================================================\n",
+      "content: The Police have arrested 15 men, aged between 26 and 56 years old, for their suspected involvement in an illegal transaction of Marine Gas Oil (MGO).\n",
+      "\n",
+      "According to data provided by Singapore’s Police, on 1st March 2024, officers from the Police Coast Guard arrested eight crew members of a Singapore-registered tugboat and another seven crew members of a foreign-registered tugboat for their suspected involvement in illegal transaction of MGO at the sea off Tuas, Singapore.\n",
+      "\n",
+      "Preliminary investigations reveal\n",
+      "====================================================================================================\n",
+      "Headline: Singapore arrests eleven men for illegal marine gas oil transaction\n",
+      "====================================================================================================\n",
+      "Details: Media sources indicated on November 4 that the Maritime and Port Authority of Singapore has arrested eleven men for their suspected involvement in an illegal transaction of marine gas oil at sea off Northern Tuas. The eleven men include six crew members of a craft of a marine service provider and another five of a foreign-registered tugboat. Preliminary reports cited that the crewmembers were thought to have misappropriated the marine gas oil and sold it to crew members of the tugboats.\n",
+      "====================================================================================================\n",
+      "bert_score: 0.5223027020692825\n",
+      "====================================================================================================\n",
+      "url: https://safety4sea.com/15-people-arrested-for-illegal-mgo-transaction-in-singapore/\n",
+      "\n",
+      "*************************************************07*************************************************\n",
+      "title: Police chase stolen tanker truck carrying 2,000 gallons of unknown liquid in Mid-Wilshire area of LA\n",
+      "====================================================================================================\n",
+      "content: Image 1 of 2 ▼\n",
+      "\n",
+      "Los Angeles police officers were in pursuit of a reported stolen tanker truck on Thursday morning.\n",
+      "\n",
+      "The Los Angeles Police Department said that around 6:30 a.m. an unknown male suspect stole a blue tanker truck from a location near the intersection of W. 6th Street and S. Curson Avenue in the Mid-Wilshire area of Los Angeles. Shortly after, police began to pursue the allegedly stolen vehicle.\n",
+      "\n",
+      "LAPD said that the stolen vehicle was carrying more than 2,000 gallons of an unknown liquid. Offi\n",
+      "====================================================================================================\n",
+      "Headline: Tanker truck carrying 2,000 gallons of unknown liquid stolen in LA\n",
+      "====================================================================================================\n",
+      "Details: According to local media sources, at around 6:30 a.m. on March 29, 2019 an unknown male suspect stole a blue tanker truck from a location near the intersection of W. 6th Street and S. Curson Avenue in the Mid-Wilshire area of Los Angeles. Shortly after, police began to pursue the allegedly stolen vehicle. Police said that the stolen vehicle was carrying more than 2,000 gallons of an unknown liquid. Officials said that the liquid was not believed to be flammable.\n",
+      "====================================================================================================\n",
+      "bert_score: 0.6466525495052338\n",
+      "====================================================================================================\n",
+      "url: https://www.fox5atlanta.com/news/police-chase-stolen-tanker-truck-carrying-2000-gallons-of-unknown-liquid-in-mid-wilshire-area-of-la\n",
+      "\n",
+      "*************************************************08*************************************************\n",
+      "title: Uber, Cabify announce they are pulling their services out of Barcelona\n",
+      "====================================================================================================\n",
+      "content: VTC vehicles block a street in Barcelona on January 19. Joan Sánchez\n",
+      "\n",
+      "Ride-hailing services Uber and Cabify announced on Thursday that they would be pulling their services out of Barcelona from tomorrow onward. The companies are doing so in response to the approval by the Catalan regional government of regulations that will force customers to request their services 15 minutes ahead of their planned journey time.\n",
+      "\n",
+      "The new law from the Generalitat, as the regional government is known, was passed in response\n",
+      "====================================================================================================\n",
+      "Headline: UPDATE - Spain: Uber and Cabify announce pulling their services out of Barcelona\n",
+      "====================================================================================================\n",
+      "Details: Ride-hailing services Uber and Cabify have announced that they will be pulling their services out of Barcelona. The companies are doing so in response to the approval by the Catalan regional government of regulations that will force customers to request their services 15 minutes ahead of their planned journey time. The new law from the Generalitat, as the regional government is known, was passed in response to demands from taxi drivers, who have been on strike in Barcelona and Madrid in recent days over w\n",
+      "====================================================================================================\n",
+      "bert_score: 0.7118738293647766\n",
+      "====================================================================================================\n",
+      "url: https://english.elpais.com/elpais/2019/01/31/inenglish/1548940738_151302.html\n",
+      "\n",
+      "*************************************************09*************************************************\n",
+      "title: 1 million pounds of smuggled pork from China seized at NJ port\n",
+      "====================================================================================================\n",
+      "content: Federal officials at the Newark port of entry seized 1 million pounds of pork products, allegedly smuggled from China, in the biggest agricultural bust in American history.\n",
+      "\n",
+      "U.S. Customs & Border Protection announced the pork seizure at a news conference on Friday at a warehouse in Elizabeth, telling NJ.com and other media outlets, the raid of more than 50 shipping containers was an effort to stop the spread of African swine fever, a virus that has decimated China's pigs.\n",
+      "\n",
+      "Anthony Bucci, an agency spokesm\n",
+      "====================================================================================================\n",
+      "Headline: US border authorities seize 1 million pounds of pork from China\n",
+      "====================================================================================================\n",
+      "Details: According to media sources, U.S. federal border agents at the Port of Newark in New Jersey seized on March 16 around 1 million pounds of pork (454 metric tons) from China, due to fears that the meat could contain traces of African swine fever which has hit Chinese pork output. \n",
+      "\n",
+      "The US Customs and Border Protection (CBP) revealed that the pork arrived in more than 50 shipping containers over the past few weeks and was hidden in containers of ramen noodles and laundry detergent packaging. The CBP believes \n",
+      "====================================================================================================\n",
+      "bert_score: 0.5224124491214752\n",
+      "====================================================================================================\n",
+      "url: https://www.northjersey.com/story/news/new-jersey/2019/03/16/one-million-pounds-smuggled-pork-china-seized-nj-port-african-swine-fever/3185268002/\n",
+      "\n",
+      "*************************************************10*************************************************\n",
+      "title: Container vessel Seaspan Lahore spills fuel at port of Algeciras\n",
+      "====================================================================================================\n",
+      "content: The captain of the port at the Port of Algeciras has detained the boxship Seaspan Lahore after a fuel spill Sunday afternoon at the Isla Verde Exterior dock.\n",
+      "\n",
+      "According to local outlet EuropaSur, the spill occurred due to a small crack in the Seaspan Lahore’s hull. It was located on the side of the vessel facing the dock, enabling containment and spill cleanup measures. The abatement effort was declared successful and is now complete.\n",
+      "\n",
+      "The vessel has been detained awaiting repairs, according EuropaSur, an\n",
+      "====================================================================================================\n",
+      "Headline: Boxship Seaspan Lahore detained following fuel spill at Port of Algeciras\n",
+      "====================================================================================================\n",
+      "Details: Sources indicated on May 25 that the boxship Seaspan Lahore has been detained by the captain of the port at the Port of Algeciras following a fuel spill on the afternoon of May 24 at the Isla Verde Exterior dock. According to sources, the spill occurred due to a crack on the vessel’s hull, which was located on the side of the vessel facing the dock, thus easing the containment and spill cleanup efforts. The vessel is detained and awaiting repairs, with the captain of the port requiring a USD 92,000 bond i\n",
+      "====================================================================================================\n",
+      "bert_score: 0.6996552348136902\n",
+      "====================================================================================================\n",
+      "url: https://cyprusshippingnews.com/2020/05/27/container-vessel-seaspan-lahore-spills-fuel-at-port-of-algeciras/\n",
+      "\n",
+      "*************************************************11*************************************************\n",
+      "title: Parts of Roxas Boulevard closed to traffic\n",
+      "====================================================================================================\n",
+      "content: Motorists should use alternate routes as the southbound lane of Roxas Boulevard from Katigbak Drive to Quirino Avenue is closed to traffic starting May 20 due to rehabilitation of the box culvert on Remedios Street.\n",
+      "\n",
+      "The Manila city government on May 19 discouraged heavy trucks from using Roxas Boulevard upon discovery of several road cracks along the major thoroughfare.\n",
+      "\n",
+      "READ: Cargo trucks discouraged from using Roxas Boulevard\n",
+      "\n",
+      "In a traffic advisory by the Manila Police District-Manila District Traffic \n",
+      "====================================================================================================\n",
+      "Headline: Cargo trucks discouraged from using Roxas Boulevard from May 20 due to closure to traffic\n",
+      "====================================================================================================\n",
+      "Details: Media sources indicated on May 20 that motorists have been advised to take alternate routes as the southbound lane of Roxas Boulevard from Katibak Drive to Quirino Avenue has been closed to traffic from May 20 due to the rehabilitation of the box culvert on Remedios Street. Manila municipal authorities on May 19 discouraged heavy cargo trucks from using Roxas Boulevard upon discovery of several road trucks along the roadway.\n",
+      "====================================================================================================\n",
+      "bert_score: 0.5179985910654068\n",
+      "====================================================================================================\n",
+      "url: https://www.portcalls.com/parts-of-roxas-boulevard-closed-to-traffic/\n",
+      "\n",
+      "*************************************************12*************************************************\n",
+      "title: CN adjusts operations amid strike at Port of Montreal\n",
+      "====================================================================================================\n",
+      "content: Canadian railway CN (NYSE: CNI) is relying on port terminals in Halifax and elsewhere on the East Coast as longshore workers continue to strike at the Port of Montreal.\n",
+      "\n",
+      "“We are monitoring the situation at the Port of Montreal closely and we are in regular contact with the responsible authorities. While we wish the parties [would] reach an agreement quickly, we are adjusting our operations in order to continue to provide service to our customers,” CN told FreightWaves.\n",
+      "\n",
+      "Those adjustments entail “leveragin\n",
+      "====================================================================================================\n",
+      "Headline: CN Railway adjusts operations amid strike at Port of Montreal\n",
+      "====================================================================================================\n",
+      "Details: Media sources indicate on August 12 that Canadian National Railway (CN) is relying on port terminals in Halifax and elsewhere on the East Coast as longshore workers continue to strike at the Port of Montreal. The railway company is by using port terminals in Halifax and elsewhere on the East Coast to continue importing and exporting goods. According to the port website, the Port of Montreal operates its own railway network, with 60 miles of track and direct access to various berths, and it has a rail inte\n",
+      "====================================================================================================\n",
+      "bert_score: 0.5962026864290237\n",
+      "====================================================================================================\n",
+      "url: https://www.freightwaves.com/news/cn-adjusts-operations-amid-strike-at-port-of-montreal\n",
+      "\n",
+      "*************************************************13*************************************************\n",
+      "title: Cocaine worth €151 million found in banana shipment destined for Hungary\n",
+      "====================================================================================================\n",
+      "content: Customs officers at the Port of Rotterdam found a massive 2,020 kilograms of cocaine hidden inside a sea container full of bananas. The shipment had arrived from Guayaquil, Ecuador, and was to continue to a company in Hungary, the Public Prosecution Service (OM) said.\n",
+      "\n",
+      "Read also Police catch dealers during cocaine party – PHOTOS, VIDEOS\n",
+      "\n",
+      "NLTimes reported that the street value of this batch of cocaine is more than 151 million EUR. The discovery was made on Saturday during an inspection at the port. Authori\n",
+      "====================================================================================================\n",
+      "Headline: Cocaine worth EUR 151 million found in banana shipment in Port of Rotterdam\n",
+      "====================================================================================================\n",
+      "Details: Media sources on June 9 that customs officers at the Port of Rotterdam found 2,020 kilograms of cocaine hidden inside a sea container full of bananas. The shipment had arrived from Guayaquil, Ecuador, and was to continue to a company in Hungary. It was reported that the street value of this batch of cocaine is more than 151 million EUR. The discovery was made on June 6 during an inspection at the port. Authorities would not identify the firm that was set to receive the shipment and stated that the recipie\n",
+      "====================================================================================================\n",
+      "bert_score: 0.6601043045520782\n",
+      "====================================================================================================\n",
+      "url: https://dailynewshungary.com/cocaine-worth-e151-million-found-in-banana-shipment-destined-for-hungary/\n",
+      "\n",
+      "*************************************************14*************************************************\n",
+      "title: 250kg of cocaine found hidden in South American helicopter parts in Spain’s Valencia\n",
+      "====================================================================================================\n",
+      "content: A GUARDIA Civil operation has resulted in the seizure of 250kg of cocaine that arrived at the Port of Valencia hidden among the pieces of a helicopter.\n",
+      "\n",
+      "The aircraft was dismantled in a South American country after being used in a campaign to prevent forest fires, and on Saturday it arrived at the port of Valencia in containers.\n",
+      "\n",
+      "Port workers took the helicopter to the APM Terminals Valencia facilities, specifically to a restricted access area controlled by security guards.\n",
+      "\n",
+      "According to investigations, a\n",
+      "====================================================================================================\n",
+      "Headline: Guardia Civil seizes 250 kg of cocaine at the Port of Valencia\n",
+      "====================================================================================================\n",
+      "Details: Spanish media sources on May 6 report that a Guardia Civil operation has resulted in the seizure of 250kg of cocaine that arrived at the Port of Valencia hidden among the pieces of a helicopter. The aircraft was dismantled in a South American country after being used in a campaign to prevent forest fires, and it arrived at the port of Valencia in containers. Port workers took the helicopter to the APM Terminals Valencia facilities, specifically to a restricted access area controlled by security guards. Th\n",
+      "====================================================================================================\n",
+      "bert_score: 0.6175416558980942\n",
+      "====================================================================================================\n",
+      "url: https://www.theolivepress.es/spain-news/2020/05/06/250kg-of-cocaine-found-hidden-in-south-american-helicopter-parts-in-spains-valencia/\n",
+      "\n",
+      "*************************************************15*************************************************\n",
+      "title: Japanese mega boxship slams into cranes at Busan New Port\n",
+      "====================================================================================================\n",
+      "content: Yet another gantry crane has been pulverised by a poorly berthed mega boxship. The Japanese controlled 13,900 teu Milano Bridge , part of the Ocean Network Express (ONE) fleet, smashed into the quayside at Busan New Port on Monday, bringing down a crane onto the back end of the boxship in the process.\n",
+      "\n",
+      "Dramatic footage sent to Splash shows the incident, which also sees the vessel come into contact with 10,000 teu boxship Seaspan Ganges.\n",
+      "\n",
+      "As well as bringing down crane 85, at least two other cranes sustain\n",
+      "====================================================================================================\n",
+      "Headline: Japanese mega boxship slams into cranes at Busan New Port\n",
+      "====================================================================================================\n",
+      "Details: Media sources indicated on April 7 that Japanese mega boxship Milano Bridge smashed into the quayside at Busan New Port on April 6, causing a crane onto the back end of the boxship in the process. Milano Bridge is part of the Ocean Network Express fleet.\n",
+      "====================================================================================================\n",
+      "bert_score: 0.6692844182252884\n",
+      "====================================================================================================\n",
+      "url: https://splash247.com/japanese-ultra-large-containership-slams-into-busan-gantry-cranes/\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "for i in range(len(df3)):\n",
+    "    row = df3.iloc[i]\n",
+    "    print(\"*\" * 49 + f\"{i + 1:02d}\" + \"*\" * 49)\n",
+    "    print_row(row)\n",
+    "    print()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "maritime",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}