{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "46322fb5-5918-4b70-9689-9e0781439ac4", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "workding dir: /Users/inflaton/code/engd/papers/maritime/global-incidents\n", "loading env vars from: /Users/inflaton/code/engd/papers/maritime/global-incidents/.env\n" ] }, { "data": { "text/plain": [ "True" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "%load_ext autoreload\n", "%autoreload 2\n", "\n", "import os\n", "import sys\n", "from pathlib import Path\n", "\n", "workding_dir = str(Path.cwd().parent)\n", "os.chdir(workding_dir)\n", "sys.path.append(workding_dir)\n", "print(\"workding dir:\", workding_dir)\n", "\n", "from dotenv import find_dotenv, load_dotenv\n", "\n", "found_dotenv = find_dotenv(\".env\")\n", "\n", "if len(found_dotenv) == 0:\n", " found_dotenv = find_dotenv(\".env.example\")\n", "print(f\"loading env vars from: {found_dotenv}\")\n", "load_dotenv(found_dotenv, override=True)" ] }, { "cell_type": "code", "execution_count": 2, "id": "daf1e3d1-75ac-4299-8bed-2f413a49f9a6", "metadata": { "tags": [] }, "outputs": [], "source": [ "import nltk\n", "from nltk.tokenize import sent_tokenize\n", "from nltk.tokenize import word_tokenize\n", "\n", "import gensim\n", "from gensim import corpora\n", "from gensim import similarities\n", "from gensim import models\n", "from gensim.models import CoherenceModel\n", "\n", "# from wordcloud import WordCloud, ImageColorGenerator\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "import pandas as pd\n", "import re\n", "import os\n", "import datetime\n", "\n", "import warnings\n", "warnings.filterwarnings('ignore')\n", "\n", "from pprint import pprint\n", "import pyLDAvis\n", "import pyLDAvis.gensim_models as gensimvis" ] }, { "cell_type": "markdown", "id": "49e6de6b-71bd-4948-8827-52601406058f", "metadata": {}, "source": [ "# Import Data" ] }, { "cell_type": "code", "execution_count": 4, "id": "49222182-7811-4fa6-8c0a-21d3a546863e", "metadata": {}, "outputs": [], "source": [ "df = pd.read_parquet('data/processed_data2.parquet')" ] }, { "cell_type": "code", "execution_count": 5, "id": "3fb59a30", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
idHeadlineDetailsSeverityCategoryRegionDatetimeYearlatlon...if_labeledMonthWeekHeadline_Detailsurltitlecontentcleaned_contentbinary_contentword_count
01Grasberg Mine- Grasberg mine workers extend st...Media sources indicate that workers at the Gra...ModerateMine Workers StrikeIndonesia28/5/17 17:082017.0-4.05608137.11302...False5.021.0Grasberg Mine- Grasberg mine workers extend st...https://news.google.com/rss/articles/CBMiZ2h0d...Freeport Indonesia mine workers extend strike ...Trucks are seen on a road in the Grasberg copp...[truck, be, see, on, road, in, grasberg, coppe...[adkerson_jakarta_try, agreement_freeport_indo...53
13Shanghai port congestion impacts terminals in ...The persisting port congestion at Shanghai’s Y...MinorPort CongestionChina27/4/17 9:162017.029.52000121.33190...False4.017.0Shanghai port congestion impacts terminals in ...https://news.google.com/rss/articles/CBMiVWh0d...Typhoon Muifa to shut China ports for second t...By Sam Whelan 13/09/2022\\n\\nAnother typhoon ha...[by, sam, whelan, typhoon, have, prompt, port,...[additional_ripple_effect, avoid_path_typhoon,...44
25UPDATE - Indonesia: Police confirm two explosi...According to local police in Jakarta, two expl...ExtremeBombing, Police OperationsIndonesia24/5/17 16:202017.0NaNNaN...True5.021.0UPDATE - Indonesia: Police confirm two explosi...https://news.google.com/rss/articles/CBMiZWh0d...Jakarta Police Receive 2 More Reports on Coldp...TEMPO.CO, Jakarta - South Jakarta Metro Police...[jakarta, south, jakarta, metro, police, recei...[actress_accord, available_day_concert, click_...24
36UPDATE - Indonesia: Severe winds damage infras...Severe winds have downed billboards and trees ...ModerateRoadway Closure / Disruption, Flooding, Severe...Indonesia19/4/17 9:102017.0-6.91264107.65700...True4.016.0UPDATE - Indonesia: Severe winds damage infras...https://news.google.com/rss/articles/CBMiSWh0d...Indonesia hit by some of strongest winds recordedA man stands near damaged houses following a t...[man, stand, near, damage, house, follow, torn...[bbc_indonesia, climatologist_government_resea...28
4142 miles E of Chesterfield - A tornado has touc...Government sources are reporting a tornado has...MinorTornadoUnited States17/9/18 19:552018.037.51000-77.61000...True9.038.02 miles E of Chesterfield - A tornado has touc...https://news.google.com/rss/articles/CBMigAFod...UPDATE: Number of homes without power down to ...More than 90,000 homes and businesses across t...[more, than, home, business, across, richmond,...[advise_seek_alternate, affect_richmond, alter...134
\n", "

5 rows × 23 columns

\n", "
" ], "text/plain": [ " id Headline \\\n", "0 1 Grasberg Mine- Grasberg mine workers extend st... \n", "1 3 Shanghai port congestion impacts terminals in ... \n", "2 5 UPDATE - Indonesia: Police confirm two explosi... \n", "3 6 UPDATE - Indonesia: Severe winds damage infras... \n", "4 14 2 miles E of Chesterfield - A tornado has touc... \n", "\n", " Details Severity \\\n", "0 Media sources indicate that workers at the Gra... Moderate \n", "1 The persisting port congestion at Shanghai’s Y... Minor \n", "2 According to local police in Jakarta, two expl... Extreme \n", "3 Severe winds have downed billboards and trees ... Moderate \n", "4 Government sources are reporting a tornado has... Minor \n", "\n", " Category Region \\\n", "0 Mine Workers Strike Indonesia \n", "1 Port Congestion China \n", "2 Bombing, Police Operations Indonesia \n", "3 Roadway Closure / Disruption, Flooding, Severe... Indonesia \n", "4 Tornado United States \n", "\n", " Datetime Year lat lon ... if_labeled Month Week \\\n", "0 28/5/17 17:08 2017.0 -4.05608 137.11302 ... False 5.0 21.0 \n", "1 27/4/17 9:16 2017.0 29.52000 121.33190 ... False 4.0 17.0 \n", "2 24/5/17 16:20 2017.0 NaN NaN ... True 5.0 21.0 \n", "3 19/4/17 9:10 2017.0 -6.91264 107.65700 ... True 4.0 16.0 \n", "4 17/9/18 19:55 2018.0 37.51000 -77.61000 ... True 9.0 38.0 \n", "\n", " Headline_Details \\\n", "0 Grasberg Mine- Grasberg mine workers extend st... \n", "1 Shanghai port congestion impacts terminals in ... \n", "2 UPDATE - Indonesia: Police confirm two explosi... \n", "3 UPDATE - Indonesia: Severe winds damage infras... \n", "4 2 miles E of Chesterfield - A tornado has touc... \n", "\n", " url \\\n", "0 https://news.google.com/rss/articles/CBMiZ2h0d... \n", "1 https://news.google.com/rss/articles/CBMiVWh0d... \n", "2 https://news.google.com/rss/articles/CBMiZWh0d... \n", "3 https://news.google.com/rss/articles/CBMiSWh0d... \n", "4 https://news.google.com/rss/articles/CBMigAFod... \n", "\n", " title \\\n", "0 Freeport Indonesia mine workers extend strike ... \n", "1 Typhoon Muifa to shut China ports for second t... \n", "2 Jakarta Police Receive 2 More Reports on Coldp... \n", "3 Indonesia hit by some of strongest winds recorded \n", "4 UPDATE: Number of homes without power down to ... \n", "\n", " content \\\n", "0 Trucks are seen on a road in the Grasberg copp... \n", "1 By Sam Whelan 13/09/2022\\n\\nAnother typhoon ha... \n", "2 TEMPO.CO, Jakarta - South Jakarta Metro Police... \n", "3 A man stands near damaged houses following a t... \n", "4 More than 90,000 homes and businesses across t... \n", "\n", " cleaned_content \\\n", "0 [truck, be, see, on, road, in, grasberg, coppe... \n", "1 [by, sam, whelan, typhoon, have, prompt, port,... \n", "2 [jakarta, south, jakarta, metro, police, recei... \n", "3 [man, stand, near, damage, house, follow, torn... \n", "4 [more, than, home, business, across, richmond,... \n", "\n", " binary_content word_count \n", "0 [adkerson_jakarta_try, agreement_freeport_indo... 53 \n", "1 [additional_ripple_effect, avoid_path_typhoon,... 44 \n", "2 [actress_accord, available_day_concert, click_... 24 \n", "3 [bbc_indonesia, climatologist_government_resea... 28 \n", "4 [advise_seek_alternate, affect_richmond, alter... 134 \n", "\n", "[5 rows x 23 columns]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head()" ] }, { "cell_type": "code", "execution_count": 6, "id": "09113e88-66cc-414c-a953-da04db83c4ae", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(3681, 23)" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.shape" ] }, { "cell_type": "markdown", "id": "037e74fc-bbcd-43e3-8346-799920cca8d8", "metadata": {}, "source": [ "# Vectorisation" ] }, { "cell_type": "markdown", "id": "d67cef3a-59fb-4dd8-adc8-2cf288b90728", "metadata": {}, "source": [ "NLP vectorization refers to the process of converting text data into numerical vectors that machine learning algorithms can understand and process. \n", "\n", "Bag-of-Words (BoW) is used here that represents text as a collection of unique words along with their frequencies. Each word is assigned an index, and the vector contains the count of each word present in the document." ] }, { "cell_type": "code", "execution_count": 7, "id": "c95b7b8a-9767-469d-812d-c9a9d9fee0e9", "metadata": {}, "outputs": [], "source": [ "df_copy = df.copy()" ] }, { "cell_type": "code", "execution_count": 8, "id": "dfb2001e-04c1-49dc-b423-a64ea47af5a9", "metadata": {}, "outputs": [], "source": [ "# choose only the extreme and severe cases for modelling\n", "cleaned = df_copy[df_copy['Severity'].isin(['Moderate'])]\n", "cleaned.reset_index(drop=True, inplace=True)" ] }, { "cell_type": "code", "execution_count": 9, "id": "de71c523-a59e-44b2-aa96-5f17d872c9c6", "metadata": {}, "outputs": [], "source": [ "headline = cleaned.binary_content" ] }, { "cell_type": "code", "execution_count": 10, "id": "5b1e34e1", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array(['accord_video', 'aftermath_shootout', 'allow_couple_flee',\n", " 'approach_truck', 'barricade_patrol_car', 'cam_footage',\n", " 'chaotic_morning_begin', 'chase_truck', 'decide_attempt',\n", " 'emergency_room_stab', 'entry_suspect_fire', 'escape_vehicle_hand',\n", " 'explain_officer_form', 'force_cop_dive', 'free_driver',\n", " 'give_police', 'greene_madison_montgomery', 'harrow_footage',\n", " 'hijack_attempt', 'hold_hostage', 'hole_truck', 'hospital_dayton',\n", " 'hostage_access_road', 'id_woman_purse', 'initial_officer_catch',\n", " 'international_airport_access', 'international_airport_london',\n", " 'late_die_morning', 'lead_officer', 'left_front_tire',\n", " 'london_police_officer', 'man_wife', 'minivan_foot_chase',\n", " 'montgomery_truck_roll', 'multiple_bullet_hole',\n", " 'name_ronald_barbara', 'negotiate_couple_release',\n", " 'numerous_bullet_hole', 'officer_stop_minivan', 'ohio_highway_cop',\n", " 'ohio_highway_patrol', 'ohio_state_highway', 'open_fire',\n", " 'original_driver_truck', 'parked_car', 'parking_lot',\n", " 'police_car_chase', 'rear_light', 'return_fire', 'road_accord',\n", " 'several_bullet', 'several_county', 'several_county_stop',\n", " 'shoot_truck', 'shut_negotiation', 'special_response_team',\n", " 'spike_strip_stop', 'stop_dayton', 'stop_till_morning',\n", " 'strip_chase', 'suspect_point_handgun', 'tactical_gear_approach',\n", " 'taylor_male_driver', 'transport_miami', 'trooper_return_fire',\n", " 'trooper_see', 'truck_drive', 'truck_drive_police',\n", " 'turn_hostage_standoff', 'tv_officer_multiple',\n", " 'unload_round_pistol', 'victim_truck',\n", " 'visible_registration_harrow', 'wbn_body', 'wbns_truck',\n", " 'white_werner'], dtype=object)" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "headline[5]" ] }, { "cell_type": "code", "execution_count": 11, "id": "677055b4-978e-4253-90f4-3f903662e225", "metadata": { "tags": [] }, "outputs": [], "source": [ "# vectorise the words\n", "doc_dict = gensim.corpora.Dictionary(headline)\n", "docs_vecs = [doc_dict.doc2bow(doc) for doc in headline]" ] }, { "cell_type": "code", "execution_count": 12, "id": "a54d1768-b069-4936-a156-deaf0b506d93", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of unique tokens: 128764\n", "Number of articles: 1761\n" ] } ], "source": [ "print('Number of unique tokens: %d' % len(doc_dict)) \n", "print('Number of articles: %d' % len(docs_vecs)) " ] }, { "cell_type": "code", "execution_count": 13, "id": "9147fa86-1503-4252-bd9b-92fea1e6a926", "metadata": { "scrolled": true, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[('heavy_rain', 134),\n", " ('supply_chain', 92),\n", " ('strong_wind', 77),\n", " ('national_weather_service', 72),\n", " ('tropical_storm', 59),\n", " ('critical_destination_port', 58),\n", " ('industrial_action', 53),\n", " ('global_supply_chain', 53),\n", " ('heavy_rainfall', 49),\n", " ('international_airport', 46),\n", " ('united_state', 46),\n", " ('high_yard_density', 43),\n", " ('social_medium', 41),\n", " ('hong_kong', 41),\n", " ('national_hurricane_center', 41),\n", " ('global_port_tracker', 39),\n", " ('global_shipping_disruption', 38),\n", " ('sign_confidence_consumer', 38),\n", " ('upgrade_import_forecast', 38),\n", " ('port_authority', 37),\n", " ('new_york_city', 36),\n", " ('schedule_reliability', 36),\n", " ('new_york', 34),\n", " ('coastal_area', 34),\n", " ('east_coast', 34),\n", " ('global_economy', 34),\n", " ('economic_growth', 33),\n", " ('mediterranean_demand', 33),\n", " ('trade_trade_statement', 33),\n", " ('high_wind', 32),\n", " ('customer_demand', 32),\n", " ('trade_statement', 32),\n", " ('severe_weather', 31),\n", " ('american_market', 31),\n", " ('relevant_information', 31),\n", " ('tropical_cyclone', 31),\n", " ('severe_thunderstorm', 31),\n", " ('several_day', 31),\n", " ('matadi_cape_town', 31),\n", " ('pacific_trade_statement', 31),\n", " ('situation_port_face', 31),\n", " ('boost_business', 30),\n", " ('important_transport_route', 30),\n", " ('industry_share_market', 30),\n", " ('part_commitment_provide', 30),\n", " ('relevant_news_hope', 30),\n", " ('state_port', 30),\n", " ('current_situation', 29),\n", " ('fourth_quarter', 29),\n", " ('shanghai_ningbo_shekou', 29),\n", " ('additional_capacity_cater', 29),\n", " ('new_zealand', 29),\n", " ('full_network', 29),\n", " ('coast_port', 28),\n", " ('cargo_move_supply', 28),\n", " ('crew_member', 28),\n", " ('storm_surge', 28),\n", " ('great_china', 28),\n", " ('many_company', 28),\n", " ('day_trade_asia', 28),\n", " ('dp_world', 27),\n", " ('international_container', 27),\n", " ('power_outage', 27),\n", " ('abijian_conakry_maputo', 27),\n", " ('america_latin', 27),\n", " ('america_trade_maersk', 27),\n", " ('arrival_trade_asia', 27),\n", " ('click_link_stay', 27),\n", " ('connect_customer_supply', 27),\n", " ('currency_depreciation_demand', 27),\n", " ('day_coast_day', 27),\n", " ('fiscal_support_consumer', 27),\n", " ('lift_outlook', 27),\n", " ('lome_onne', 27),\n", " ('monthly_please', 27),\n", " ('pacific_america_trade', 27),\n", " ('pointe_noire_balboa', 27),\n", " ('post_subscribe_asia', 27),\n", " ('professional_find_market', 27),\n", " ('question_supply_chain', 27),\n", " ('service_china', 27),\n", " ('useful_subscribe_maersk', 27),\n", " ('help_business', 26),\n", " ('new_jersey', 26),\n", " ('port_congestion', 26),\n", " ('asia_trade_statement', 26),\n", " ('america_trade_statement', 26),\n", " ('international_longshore_warehouse', 25),\n", " ('next_day', 25),\n", " ('many_country', 25),\n", " ('meet_firm', 25),\n", " ('website_see_service', 25),\n", " ('situation_trade', 25),\n", " ('vietnam_cambodia_myanmar', 25),\n", " ('main_port', 25),\n", " ('high_inflation', 24),\n", " ('port_situation', 24),\n", " ('high_level', 24),\n", " ('china_area', 24),\n", " ('passenger_service', 23)]\n" ] } ], "source": [ "# Calculate word frequencies\n", "word_frequencies = {doc_dict[word_id]: freq for word_id, freq in doc_dict.cfs.items()}\n", "sorted_words = sorted(word_frequencies.items(), key=lambda x: x[1], reverse=True)\n", "\n", "pprint(sorted_words[:100])" ] }, { "cell_type": "markdown", "id": "5ed78239-2ce1-4784-a8f4-4c7438c8627b", "metadata": {}, "source": [ "# LDA Modelling" ] }, { "cell_type": "markdown", "id": "9db83273-461d-4f70-b23f-ec967579d94f", "metadata": {}, "source": [ "## Benchmark Model" ] }, { "cell_type": "code", "execution_count": 15, "id": "e6d577bd-9936-4d45-be90-345af2eb4827", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 1.32 s, sys: 837 ms, total: 2.15 s\n", "Wall time: 1min 21s\n" ] } ], "source": [ "%%time\n", "\n", "# Build LDA benchmark model\n", "lda_model = gensim.models.LdaMulticore(corpus=docs_vecs,\n", " id2word=doc_dict,\n", " num_topics=4, \n", " random_state=42,\n", " chunksize=100,\n", " passes=10,\n", " per_word_topics=True)" ] }, { "cell_type": "code", "execution_count": 16, "id": "c4f1521f-5f43-40d2-a3a3-a8ac2ca6fec2", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[(0,\n", " '0.000*\"heavy_rain\" + 0.000*\"tropical_storm\" + '\n", " '0.000*\"national_weather_service\" + 0.000*\"strong_wind\" + '\n", " '0.000*\"average_day\" + 0.000*\"united_state\" + 0.000*\"supply_chain\" + '\n", " '0.000*\"port_congestion\" + 0.000*\"heavy_rainfall\" + '\n", " '0.000*\"national_hurricane_center\"'),\n", " (1,\n", " '0.000*\"global_port_tracker\" + 0.000*\"global_shipping_disruption\" + '\n", " '0.000*\"upgrade_import_forecast\" + 0.000*\"sign_confidence_consumer\" + '\n", " '0.000*\"supply_chain\" + 0.000*\"american_market\" + '\n", " '0.000*\"relevant_information\" + 0.000*\"relevant_news_hope\" + '\n", " '0.000*\"state_port\" + 0.000*\"industry_share_market\"'),\n", " (2,\n", " '0.001*\"heavy_rain\" + 0.000*\"supply_chain\" + 0.000*\"united_state\" + '\n", " '0.000*\"critical_destination_port\" + 0.000*\"national_weather_service\" + '\n", " '0.000*\"strong_wind\" + 0.000*\"high_yard_density\" + '\n", " '0.000*\"mediterranean_demand\" + 0.000*\"pacific_trade_statement\" + '\n", " '0.000*\"trade_trade_statement\"'),\n", " (3,\n", " '0.000*\"heavy_rain\" + 0.000*\"critical_destination_port\" + '\n", " '0.000*\"global_supply_chain\" + 0.000*\"trade_statement\" + '\n", " '0.000*\"schedule_reliability\" + 0.000*\"day_trade_asia\" + '\n", " '0.000*\"full_network\" + 0.000*\"full_truck\" + 0.000*\"january_trade_maersk\" + '\n", " '0.000*\"stable_network\"')]\n" ] } ], "source": [ "from pprint import pprint\n", "\n", "# Print the Keyword in the 10 topics\n", "pprint(lda_model.print_topics())\n", "doc_lda = lda_model[docs_vecs]" ] }, { "cell_type": "code", "execution_count": 17, "id": "fd57b1f4-a6cd-41e8-964f-d8a1d30aa3c9", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Coherence Score LDAModel: 0.5135998374037359\n" ] } ], "source": [ "# Compute Benchmark Coherence Score\n", "coherence_model_lda = CoherenceModel(model=lda_model, texts=headline, dictionary=doc_dict, coherence='c_v')\n", "coherence_lda = coherence_model_lda.get_coherence()\n", "print('\\nCoherence Score LDAModel: ', coherence_lda)" ] }, { "cell_type": "code", "execution_count": 18, "id": "152e5a3a-7afe-4fb8-a02f-d7492ad80936", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "Perplexity for LDAModel: -11.69195177074239\n" ] } ], "source": [ "# Compute Benchmark Perplexity\n", "perplex= lda_model.log_perplexity(docs_vecs, total_docs=None) #For LDAModel\n", " # a measure of how good the model is. lower the better.\n", "\n", "print('\\nPerplexity for LDAModel: ', perplex)" ] }, { "cell_type": "code", "execution_count": 20, "id": "7dd3a60a-5c6f-4249-9868-30528a5b0ac8", "metadata": {}, "outputs": [], "source": [ "from pprint import pprint\n", "import pyLDAvis\n", "import pyLDAvis.gensim_models as gensimvis\n", "\n", "# feed the LDA model into the pyLDAvis instance\n", "pyLDAvis.enable_notebook()\n", "visual= gensimvis.prepare(lda_model, docs_vecs, doc_dict)\n", "\n", "# Save the output to the html file\n", "pyLDAvis.save_html(visual, \"data/topic_viz_benchmark_moderate.html\")" ] }, { "cell_type": "code", "execution_count": 21, "id": "3a5612f7-6358-49c8-aba9-8aa54e275c6f", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Topic KeywordsTopic ID
00.000*\"heavy_rain\" + 0.000*\"tropical_storm\" + 0.000*\"national_weather_service\" + 0.000*\"strong_wind\" + 0.000*\"average_day\" + 0.000*\"united_state\"0
10.000*\"global_port_tracker\" + 0.000*\"global_shipping_disruption\" + 0.000*\"upgrade_import_forecast\" + 0.000*\"sign_confidence_consumer\" + 0.000*\"supply_chain\" + 0.000*\"american_market\"1
20.001*\"heavy_rain\" + 0.000*\"supply_chain\" + 0.000*\"united_state\" + 0.000*\"critical_destination_port\" + 0.000*\"national_weather_service\" + 0.000*\"strong_wind\"2
30.000*\"heavy_rain\" + 0.000*\"critical_destination_port\" + 0.000*\"global_supply_chain\" + 0.000*\"trade_statement\" + 0.000*\"schedule_reliability\" + 0.000*\"day_trade_asia\"3
\n", "
" ], "text/plain": [ " Topic Keywords \\\n", "0 0.000*\"heavy_rain\" + 0.000*\"tropical_storm\" + 0.000*\"national_weather_service\" + 0.000*\"strong_wind\" + 0.000*\"average_day\" + 0.000*\"united_state\" \n", "1 0.000*\"global_port_tracker\" + 0.000*\"global_shipping_disruption\" + 0.000*\"upgrade_import_forecast\" + 0.000*\"sign_confidence_consumer\" + 0.000*\"supply_chain\" + 0.000*\"american_market\" \n", "2 0.001*\"heavy_rain\" + 0.000*\"supply_chain\" + 0.000*\"united_state\" + 0.000*\"critical_destination_port\" + 0.000*\"national_weather_service\" + 0.000*\"strong_wind\" \n", "3 0.000*\"heavy_rain\" + 0.000*\"critical_destination_port\" + 0.000*\"global_supply_chain\" + 0.000*\"trade_statement\" + 0.000*\"schedule_reliability\" + 0.000*\"day_trade_asia\" \n", "\n", " Topic ID \n", "0 0 \n", "1 1 \n", "2 2 \n", "3 3 " ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pd.set_option('max_colwidth', 200)\n", "# Get the topics and their top keywords into a dataframe\n", "topics = lda_model.show_topics(num_words=6) \n", "\n", "topic_keywords = pd.DataFrame()\n", "for topic_id, topic in topics:\n", " topic_keywords.at[topic_id, 'Topic Keywords'] = topic\n", "\n", "topic_keywords['Topic ID'] = topic_keywords.index\n", "# topic_keywords['Topic Name'] = topic_mapping \n", "topic_keywords" ] }, { "cell_type": "code", "execution_count": 22, "id": "26da4eea-06a0-4ff7-ae14-2f40fa0db04b", "metadata": {}, "outputs": [], "source": [ "# break " ] }, { "cell_type": "markdown", "id": "1895598f-3e5f-4acd-83a6-4491cc90f695", "metadata": {}, "source": [ "# Hyper-Perameter Tuning and Evaluation" ] }, { "cell_type": "markdown", "id": "47136c89-ff7b-4ac9-840f-04122fe62160", "metadata": {}, "source": [ "Run the cells below only for re-modelling with new datasets, the whole tuning and evaluation process may take hours to run." ] }, { "cell_type": "code", "execution_count": 23, "id": "c79ca5c4-e078-43ce-a430-8c1ed93dcd64", "metadata": {}, "outputs": [], "source": [ "# hyper-perameter tuning (alpha and beta)\n", "def compute_coherence_values(corpus, dictionary, k, a, b):\n", " \n", " lda_model = gensim.models.LdaMulticore(corpus=corpus,\n", " id2word=dictionary,\n", " num_topics=k, \n", " random_state=42,\n", " chunksize=100,\n", " passes=10,\n", " alpha=a,\n", " eta=b)\n", " \n", " coherence_model_lda = CoherenceModel(model=lda_model, texts=headline, dictionary=doc_dict, coherence='c_v')\n", " coherence = coherence_model_lda.get_coherence()\n", " perplex = lda_model.log_perplexity(docs_vecs, total_docs=None) \n", " \n", " return coherence, perplex" ] }, { "cell_type": "code", "execution_count": 24, "id": "1c3c8478-9336-40f2-bb30-a37db4243b67", "metadata": {}, "outputs": [], "source": [ "# setup\n", "import numpy as np\n", "\n", "from gensim.models import CoherenceModel\n", "\n", "model_list = []\n", "coherence_values = []\n", "perplexity_values = []\n", "model_topics = []\n", "alpha_result = []\n", "beta_result = []\n", "\n", "# topic ranges\n", "num_topics = range(4, 13)\n", "\n", "# Alpha parameter\n", "alpha = list(np.arange(0.31, 1, 0.3))\n", "alpha.append('symmetric')\n", "alpha.append('asymmetric')\n", "\n", "# Beta parameter\n", "beta = list(np.arange(0.31, 1, 0.3))\n", "beta.append('symmetric')" ] }, { "cell_type": "markdown", "id": "c7e6bc53-0b57-4858-879a-644eca54ddbc", "metadata": {}, "source": [ "Rational behind the alpha and eta: https://stats.stackexchange.com/questions/37405/natural-interpretation-for-lda-hyperparameters" ] }, { "cell_type": "code", "execution_count": 25, "id": "02877b81-32df-4168-8e62-4cbca2be100b", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Topic range: range(4, 13)\n", "Alpha: [0.31, 0.61, 0.9099999999999999, 'symmetric', 'asymmetric']\n", "Beta: [0.31, 0.61, 0.9099999999999999, 'symmetric']\n" ] } ], "source": [ "print(\"Topic range: \",num_topics)\n", "print(\"Alpha: \",alpha)\n", "print(\"Beta: \", beta)" ] }, { "cell_type": "code", "execution_count": 26, "id": "3c1f703c-4778-467f-a12e-0c18eeb274c5", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2024-06-30 15:33:21.408395\n", "#Topics: 4, CV Score: 0.5164676429141806, PV Score: -11.613380568733923, Alpha: 0.31, Beta: 0.31\n", "#Topics: 5, CV Score: 0.5683002282838892, PV Score: -11.563534433528984, Alpha: 0.31, Beta: 0.31\n", "#Topics: 6, CV Score: 0.46555259667124016, PV Score: -11.53219474749234, Alpha: 0.31, Beta: 0.31\n", "#Topics: 7, CV Score: 0.5159944038644447, PV Score: -11.50783852707268, Alpha: 0.31, Beta: 0.31\n", "#Topics: 8, CV Score: 0.4639539916466071, PV Score: -11.49178161514173, Alpha: 0.31, Beta: 0.31\n", "#Topics: 9, CV Score: 0.6087247241979915, PV Score: -11.474821405795332, Alpha: 0.31, Beta: 0.31\n", "#Topics: 10, CV Score: 0.5444179957484441, PV Score: -11.469740741939871, Alpha: 0.31, Beta: 0.31\n", "#Topics: 11, CV Score: 0.5694955423900976, PV Score: -11.473400251077415, Alpha: 0.31, Beta: 0.31\n", "#Topics: 12, CV Score: 0.49528605221915506, PV Score: -11.464966921298775, Alpha: 0.31, Beta: 0.31\n", "#Topics: 4, CV Score: 0.4846077269482417, PV Score: -11.615188474292983, Alpha: 0.31, Beta: 0.61\n", "#Topics: 5, CV Score: 0.40044534503210266, PV Score: -11.571275497697407, Alpha: 0.31, Beta: 0.61\n", "#Topics: 6, CV Score: 0.47710647516703003, PV Score: -11.542620661350824, Alpha: 0.31, Beta: 0.61\n", "#Topics: 7, CV Score: 0.41352955820583087, PV Score: -11.540942420187077, Alpha: 0.31, Beta: 0.61\n", "#Topics: 8, CV Score: 0.4289987234137122, PV Score: -11.52253612455723, Alpha: 0.31, Beta: 0.61\n", "#Topics: 9, CV Score: 0.4870040213227611, PV Score: -11.517891900057037, Alpha: 0.31, Beta: 0.61\n", "#Topics: 10, CV Score: 0.550960377654979, PV Score: -11.515178838931973, Alpha: 0.31, Beta: 0.61\n", "#Topics: 11, CV Score: 0.54629246756211, PV Score: -11.521576629251783, Alpha: 0.31, Beta: 0.61\n", "#Topics: 12, CV Score: 0.4734093628757035, PV Score: -11.663901784573016, Alpha: 0.31, Beta: 0.61\n", "#Topics: 4, CV Score: 0.5226264223935461, PV Score: -11.581600670664274, Alpha: 0.31, Beta: 0.9099999999999999\n", "#Topics: 5, CV Score: 0.4815626090375563, PV Score: -11.57887038938498, Alpha: 0.31, Beta: 0.9099999999999999\n", "#Topics: 6, CV Score: 0.4822722108681728, PV Score: -11.559852785176927, Alpha: 0.31, Beta: 0.9099999999999999\n", "#Topics: 7, CV Score: 0.5190214814534763, PV Score: -11.58883143718404, Alpha: 0.31, Beta: 0.9099999999999999\n", "#Topics: 8, CV Score: 0.45997514950408147, PV Score: -11.55820866243133, Alpha: 0.31, Beta: 0.9099999999999999\n", "#Topics: 9, CV Score: 0.36374367474380254, PV Score: -11.63640550749839, Alpha: 0.31, Beta: 0.9099999999999999\n", "#Topics: 10, CV Score: 0.5643564534964567, PV Score: -11.557503103730841, Alpha: 0.31, Beta: 0.9099999999999999\n", "#Topics: 11, CV Score: 0.5469185375162117, PV Score: -11.5647045020449, Alpha: 0.31, Beta: 0.9099999999999999\n", "#Topics: 12, CV Score: 0.5792875415068292, PV Score: -11.806654395764777, Alpha: 0.31, Beta: 0.9099999999999999\n", "#Topics: 4, CV Score: 0.49804879882635616, PV Score: -11.642897316807662, Alpha: 0.31, Beta: symmetric\n", "#Topics: 5, CV Score: 0.5778525494321111, PV Score: -11.619415198399734, Alpha: 0.31, Beta: symmetric\n", "#Topics: 6, CV Score: 0.46263841554243873, PV Score: -11.603945568112582, Alpha: 0.31, Beta: symmetric\n", "#Topics: 7, CV Score: 0.4751517261203773, PV Score: -11.610468303646345, Alpha: 0.31, Beta: symmetric\n", "#Topics: 8, CV Score: 0.3914935846000445, PV Score: -11.642292032598931, Alpha: 0.31, Beta: symmetric\n", "#Topics: 9, CV Score: 0.6211150499938938, PV Score: -11.582063142378686, Alpha: 0.31, Beta: symmetric\n", "#Topics: 10, CV Score: 0.6020085770917671, PV Score: -11.583284044840946, Alpha: 0.31, Beta: symmetric\n", "#Topics: 11, CV Score: 0.6175933853797404, PV Score: -11.59165919505012, Alpha: 0.31, Beta: symmetric\n", "#Topics: 12, CV Score: 0.5908368128582339, PV Score: -11.564916248235656, Alpha: 0.31, Beta: symmetric\n", "#Topics: 4, CV Score: 0.5003561625277873, PV Score: -11.637393333313193, Alpha: 0.61, Beta: 0.31\n", "#Topics: 5, CV Score: 0.5317007800262091, PV Score: -11.641278862591506, Alpha: 0.61, Beta: 0.31\n", "#Topics: 6, CV Score: 0.557100339849869, PV Score: -11.584370827514544, Alpha: 0.61, Beta: 0.31\n", "#Topics: 7, CV Score: 0.5172402989720184, PV Score: -11.566723792196806, Alpha: 0.61, Beta: 0.31\n", "#Topics: 8, CV Score: 0.4886338324086602, PV Score: -11.54297247369616, Alpha: 0.61, Beta: 0.31\n", "#Topics: 9, CV Score: 0.5747517437985423, PV Score: -11.543691538358624, Alpha: 0.61, Beta: 0.31\n", "#Topics: 10, CV Score: 0.6423710177739427, PV Score: -11.54859461537098, Alpha: 0.61, Beta: 0.31\n", "#Topics: 11, CV Score: 0.5144410257534869, PV Score: -11.73660202888069, Alpha: 0.61, Beta: 0.31\n", "#Topics: 12, CV Score: 0.5717578206426642, PV Score: -11.530830842170962, Alpha: 0.61, Beta: 0.31\n", "#Topics: 4, CV Score: 0.5180085860214778, PV Score: -11.597568405435847, Alpha: 0.61, Beta: 0.61\n", "#Topics: 5, CV Score: 0.48198140101849224, PV Score: -11.580472178461592, Alpha: 0.61, Beta: 0.61\n", "#Topics: 6, CV Score: 0.5224885861991673, PV Score: -11.607306821970512, Alpha: 0.61, Beta: 0.61\n", "#Topics: 7, CV Score: 0.5116028282066244, PV Score: -11.566861788342026, Alpha: 0.61, Beta: 0.61\n", "#Topics: 8, CV Score: 0.4497755401591277, PV Score: -11.575814895292964, Alpha: 0.61, Beta: 0.61\n", "#Topics: 9, CV Score: 0.5327566529438476, PV Score: -11.586548862112604, Alpha: 0.61, Beta: 0.61\n", "#Topics: 10, CV Score: 0.5214711529865992, PV Score: -11.628991593567058, Alpha: 0.61, Beta: 0.61\n", "#Topics: 11, CV Score: 0.6139640248470433, PV Score: -11.594641144555219, Alpha: 0.61, Beta: 0.61\n", "#Topics: 12, CV Score: 0.5356515378529471, PV Score: -11.59298627864858, Alpha: 0.61, Beta: 0.61\n", "#Topics: 4, CV Score: 0.49160818384314564, PV Score: -11.60400992430138, Alpha: 0.61, Beta: 0.9099999999999999\n", "#Topics: 5, CV Score: 0.489468146274511, PV Score: -11.596778903516313, Alpha: 0.61, Beta: 0.9099999999999999\n", "#Topics: 6, CV Score: 0.47456800821881234, PV Score: -11.59473954643288, Alpha: 0.61, Beta: 0.9099999999999999\n", "#Topics: 7, CV Score: 0.5576401494669467, PV Score: -11.594596066878076, Alpha: 0.61, Beta: 0.9099999999999999\n", "#Topics: 8, CV Score: 0.4667184261166916, PV Score: -11.60378701602699, Alpha: 0.61, Beta: 0.9099999999999999\n", "#Topics: 9, CV Score: 0.5695086900271756, PV Score: -11.609075892863483, Alpha: 0.61, Beta: 0.9099999999999999\n", "#Topics: 10, CV Score: 0.6005670921966928, PV Score: -11.614071327825993, Alpha: 0.61, Beta: 0.9099999999999999\n", "#Topics: 11, CV Score: 0.5480401775800159, PV Score: -11.623267686418437, Alpha: 0.61, Beta: 0.9099999999999999\n", "#Topics: 12, CV Score: 0.5396539221868254, PV Score: -11.712120797560939, Alpha: 0.61, Beta: 0.9099999999999999\n", "#Topics: 4, CV Score: 0.45957893891496127, PV Score: -11.6729179593589, Alpha: 0.61, Beta: symmetric\n", "#Topics: 5, CV Score: 0.5497056258415677, PV Score: -11.65337679958747, Alpha: 0.61, Beta: symmetric\n", "#Topics: 6, CV Score: 0.5704158041118883, PV Score: -11.645450690343361, Alpha: 0.61, Beta: symmetric\n", "#Topics: 7, CV Score: 0.5268609558304878, PV Score: -11.638214852770608, Alpha: 0.61, Beta: symmetric\n", "#Topics: 8, CV Score: 0.4658802544250874, PV Score: -11.63919464483393, Alpha: 0.61, Beta: symmetric\n", "#Topics: 9, CV Score: 0.6506499863493337, PV Score: -11.63849523064737, Alpha: 0.61, Beta: symmetric\n", "#Topics: 10, CV Score: 0.649136503615956, PV Score: -11.616778698707527, Alpha: 0.61, Beta: symmetric\n", "#Topics: 11, CV Score: 0.6461235115651532, PV Score: -11.660982724024588, Alpha: 0.61, Beta: symmetric\n", "#Topics: 12, CV Score: 0.5795686612759658, PV Score: -11.654057997902278, Alpha: 0.61, Beta: symmetric\n", "#Topics: 4, CV Score: 0.43540817795833214, PV Score: -11.686659200023914, Alpha: 0.9099999999999999, Beta: 0.31\n", "#Topics: 5, CV Score: 0.5462127792495466, PV Score: -11.62457497965543, Alpha: 0.9099999999999999, Beta: 0.31\n", "#Topics: 6, CV Score: 0.5524384036237932, PV Score: -11.604354138411734, Alpha: 0.9099999999999999, Beta: 0.31\n", "#Topics: 7, CV Score: 0.5902480026727759, PV Score: -11.588582708471723, Alpha: 0.9099999999999999, Beta: 0.31\n", "#Topics: 8, CV Score: 0.47181028096737715, PV Score: -11.583687892570778, Alpha: 0.9099999999999999, Beta: 0.31\n", "#Topics: 9, CV Score: 0.6963152362329474, PV Score: -11.581137216572229, Alpha: 0.9099999999999999, Beta: 0.31\n", "#Topics: 10, CV Score: 0.6356546949964338, PV Score: -11.583413852955387, Alpha: 0.9099999999999999, Beta: 0.31\n", "#Topics: 11, CV Score: 0.6110954828509537, PV Score: -11.578235683186872, Alpha: 0.9099999999999999, Beta: 0.31\n", "#Topics: 12, CV Score: 0.5948748370571434, PV Score: -11.591490601197242, Alpha: 0.9099999999999999, Beta: 0.31\n", "#Topics: 4, CV Score: 0.48818167146918845, PV Score: -11.61574038870984, Alpha: 0.9099999999999999, Beta: 0.61\n", "#Topics: 5, CV Score: 0.4509752119451145, PV Score: -11.604195597608339, Alpha: 0.9099999999999999, Beta: 0.61\n", "#Topics: 6, CV Score: 0.35645184464437624, PV Score: -11.637862647568788, Alpha: 0.9099999999999999, Beta: 0.61\n", "#Topics: 7, CV Score: 0.526882700494322, PV Score: -11.653081895856653, Alpha: 0.9099999999999999, Beta: 0.61\n", "#Topics: 8, CV Score: 0.42882071728968496, PV Score: -11.609087232613648, Alpha: 0.9099999999999999, Beta: 0.61\n", "#Topics: 9, CV Score: 0.6693854566085815, PV Score: -11.614610403392634, Alpha: 0.9099999999999999, Beta: 0.61\n", "#Topics: 10, CV Score: 0.6179132712789255, PV Score: -11.623350711437743, Alpha: 0.9099999999999999, Beta: 0.61\n", "#Topics: 11, CV Score: 0.6085517947565203, PV Score: -11.63547085027884, Alpha: 0.9099999999999999, Beta: 0.61\n", "#Topics: 12, CV Score: 0.5577009173064904, PV Score: -11.651522052657057, Alpha: 0.9099999999999999, Beta: 0.61\n", "#Topics: 4, CV Score: 0.5230671592592115, PV Score: -11.623717800797593, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n", "#Topics: 5, CV Score: 0.42768193852006214, PV Score: -11.62144694067679, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n", "#Topics: 6, CV Score: 0.5364419205784232, PV Score: -11.624175638013885, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n", "#Topics: 7, CV Score: 0.4857096769423311, PV Score: -11.63428665005544, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n", "#Topics: 8, CV Score: 0.5044375689801471, PV Score: -11.642138170613716, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n", "#Topics: 9, CV Score: 0.5696949744243359, PV Score: -11.654881948442265, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n", "#Topics: 10, CV Score: 0.6326338442828181, PV Score: -11.66680515031184, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n", "#Topics: 11, CV Score: 0.5727395073356502, PV Score: -11.69273499062231, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n", "#Topics: 12, CV Score: 0.559542950238316, PV Score: -11.748344593064868, Alpha: 0.9099999999999999, Beta: 0.9099999999999999\n", "#Topics: 4, CV Score: 0.3804434166795542, PV Score: -11.768219382154312, Alpha: 0.9099999999999999, Beta: symmetric\n", "#Topics: 5, CV Score: 0.5346775324429552, PV Score: -11.668832927089504, Alpha: 0.9099999999999999, Beta: symmetric\n", "#Topics: 6, CV Score: 0.35761369835980755, PV Score: -11.7234783761902, Alpha: 0.9099999999999999, Beta: symmetric\n", "#Topics: 7, CV Score: 0.5739180802234388, PV Score: -11.650366733021364, Alpha: 0.9099999999999999, Beta: symmetric\n", "#Topics: 8, CV Score: 0.4690450833134745, PV Score: -11.671911408908096, Alpha: 0.9099999999999999, Beta: symmetric\n", "#Topics: 9, CV Score: 0.6456959957990995, PV Score: -11.692264722739667, Alpha: 0.9099999999999999, Beta: symmetric\n", "#Topics: 10, CV Score: 0.6358178052250946, PV Score: -11.694376238908927, Alpha: 0.9099999999999999, Beta: symmetric\n", "#Topics: 11, CV Score: 0.6545497380387766, PV Score: -11.728265629814926, Alpha: 0.9099999999999999, Beta: symmetric\n", "#Topics: 12, CV Score: 0.5668227847904856, PV Score: -11.831189552283824, Alpha: 0.9099999999999999, Beta: symmetric\n", "#Topics: 4, CV Score: 0.5079465498144028, PV Score: -11.744019308081587, Alpha: symmetric, Beta: 0.31\n", "#Topics: 5, CV Score: 0.5207685525937951, PV Score: -11.56250304658784, Alpha: symmetric, Beta: 0.31\n", "#Topics: 6, CV Score: 0.4520436901506503, PV Score: -11.517440615925524, Alpha: symmetric, Beta: 0.31\n", "#Topics: 7, CV Score: 0.3984484028061592, PV Score: -11.49927289529406, Alpha: symmetric, Beta: 0.31\n", "#Topics: 8, CV Score: 0.46282464845379434, PV Score: -11.447392562926892, Alpha: symmetric, Beta: 0.31\n", "#Topics: 9, CV Score: 0.5628759566317312, PV Score: -11.431744800328639, Alpha: symmetric, Beta: 0.31\n", "#Topics: 10, CV Score: 0.5550416515331734, PV Score: -11.414885267043056, Alpha: symmetric, Beta: 0.31\n", "#Topics: 11, CV Score: 0.5292928552625016, PV Score: -11.412137160368818, Alpha: symmetric, Beta: 0.31\n", "#Topics: 12, CV Score: 0.4300247772463878, PV Score: -11.42635756927043, Alpha: symmetric, Beta: 0.31\n", "#Topics: 4, CV Score: 0.551938709076145, PV Score: -11.646172984411194, Alpha: symmetric, Beta: 0.61\n", "#Topics: 5, CV Score: 0.4809237301776637, PV Score: -11.559942575549936, Alpha: symmetric, Beta: 0.61\n", "#Topics: 6, CV Score: 0.46228300273399897, PV Score: -11.515382800949316, Alpha: symmetric, Beta: 0.61\n", "#Topics: 7, CV Score: 0.46287366351823206, PV Score: -11.4950582564519, Alpha: symmetric, Beta: 0.61\n", "#Topics: 8, CV Score: 0.4606576501635787, PV Score: -11.486398241110296, Alpha: symmetric, Beta: 0.61\n", "#Topics: 9, CV Score: 0.4481608851641272, PV Score: -11.520590899826077, Alpha: symmetric, Beta: 0.61\n", "#Topics: 10, CV Score: 0.5543837883238459, PV Score: -11.57085700936084, Alpha: symmetric, Beta: 0.61\n", "#Topics: 11, CV Score: 0.5316944180923253, PV Score: -11.578237826778107, Alpha: symmetric, Beta: 0.61\n", "#Topics: 12, CV Score: 0.4687550311670818, PV Score: -11.453118532016195, Alpha: symmetric, Beta: 0.61\n", "#Topics: 4, CV Score: 0.5365308671952822, PV Score: -11.58505859292843, Alpha: symmetric, Beta: 0.9099999999999999\n", "#Topics: 5, CV Score: 0.4573271215291844, PV Score: -11.55400501711275, Alpha: symmetric, Beta: 0.9099999999999999\n", "#Topics: 6, CV Score: 0.4845637557935425, PV Score: -11.538682634708005, Alpha: symmetric, Beta: 0.9099999999999999\n", "#Topics: 7, CV Score: 0.43505334283558816, PV Score: -11.560310334374622, Alpha: symmetric, Beta: 0.9099999999999999\n", "#Topics: 8, CV Score: 0.48972094471582783, PV Score: -11.52262741120793, Alpha: symmetric, Beta: 0.9099999999999999\n", "#Topics: 9, CV Score: 0.5105059500845196, PV Score: -11.51304907591031, Alpha: symmetric, Beta: 0.9099999999999999\n", "#Topics: 10, CV Score: 0.5591753304786676, PV Score: -11.5054622260105, Alpha: symmetric, Beta: 0.9099999999999999\n", "#Topics: 11, CV Score: 0.5528861076804188, PV Score: -11.504385112555493, Alpha: symmetric, Beta: 0.9099999999999999\n", "#Topics: 12, CV Score: 0.49367285382840453, PV Score: -11.500990267519892, Alpha: symmetric, Beta: 0.9099999999999999\n", "#Topics: 4, CV Score: 0.46594738594111, PV Score: -11.638161524533613, Alpha: symmetric, Beta: symmetric\n", "#Topics: 5, CV Score: 0.5212817101806027, PV Score: -11.605645781016982, Alpha: symmetric, Beta: symmetric\n", "#Topics: 6, CV Score: 0.44807641083986144, PV Score: -11.582140562178676, Alpha: symmetric, Beta: symmetric\n", "#Topics: 7, CV Score: 0.509705458396568, PV Score: -11.559129856046134, Alpha: symmetric, Beta: symmetric\n", "#Topics: 8, CV Score: 0.45072428914784013, PV Score: -11.521973552776846, Alpha: symmetric, Beta: symmetric\n", "#Topics: 9, CV Score: 0.6163956153172733, PV Score: -11.532030259535352, Alpha: symmetric, Beta: symmetric\n", "#Topics: 10, CV Score: 0.5731211242304707, PV Score: -11.526033204967101, Alpha: symmetric, Beta: symmetric\n", "#Topics: 11, CV Score: 0.6000508615312063, PV Score: -11.49509548343454, Alpha: symmetric, Beta: symmetric\n", "#Topics: 12, CV Score: 0.4536127133023553, PV Score: -11.499639773335504, Alpha: symmetric, Beta: symmetric\n", "#Topics: 4, CV Score: 0.5517238147309061, PV Score: -11.613287330943077, Alpha: asymmetric, Beta: 0.31\n", "#Topics: 5, CV Score: 0.653586278160635, PV Score: -11.544483709714015, Alpha: asymmetric, Beta: 0.31\n", "#Topics: 6, CV Score: 0.5297708553518212, PV Score: -11.565107511094201, Alpha: asymmetric, Beta: 0.31\n", "#Topics: 7, CV Score: 0.4308728891563382, PV Score: -11.482917171438388, Alpha: asymmetric, Beta: 0.31\n", "#Topics: 8, CV Score: 0.5225479879550474, PV Score: -11.452961250714646, Alpha: asymmetric, Beta: 0.31\n", "#Topics: 9, CV Score: 0.6266251392086057, PV Score: -11.436045988851452, Alpha: asymmetric, Beta: 0.31\n", "#Topics: 10, CV Score: 0.6244003133589429, PV Score: -11.420447490550442, Alpha: asymmetric, Beta: 0.31\n" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "File \u001b[0;32m:10\u001b[0m\n", "Cell \u001b[0;32mIn[23], line 4\u001b[0m, in \u001b[0;36mcompute_coherence_values\u001b[0;34m(corpus, dictionary, k, a, b)\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcompute_coherence_values\u001b[39m(corpus, dictionary, k, a, b):\n\u001b[0;32m----> 4\u001b[0m lda_model \u001b[38;5;241m=\u001b[39m \u001b[43mgensim\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodels\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mLdaMulticore\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcorpus\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcorpus\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[43mid2word\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdictionary\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[43m \u001b[49m\u001b[43mnum_topics\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43mrandom_state\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m42\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 8\u001b[0m \u001b[43m \u001b[49m\u001b[43mchunksize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m100\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 9\u001b[0m \u001b[43m \u001b[49m\u001b[43mpasses\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 10\u001b[0m \u001b[43m \u001b[49m\u001b[43malpha\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43ma\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 11\u001b[0m \u001b[43m \u001b[49m\u001b[43meta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 13\u001b[0m coherence_model_lda \u001b[38;5;241m=\u001b[39m CoherenceModel(model\u001b[38;5;241m=\u001b[39mlda_model, texts\u001b[38;5;241m=\u001b[39mheadline, dictionary\u001b[38;5;241m=\u001b[39mdoc_dict, coherence\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mc_v\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 14\u001b[0m coherence \u001b[38;5;241m=\u001b[39m coherence_model_lda\u001b[38;5;241m.\u001b[39mget_coherence()\n", "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/models/ldamulticore.py:186\u001b[0m, in \u001b[0;36mLdaMulticore.__init__\u001b[0;34m(self, corpus, num_topics, id2word, workers, chunksize, passes, batch, alpha, eta, decay, offset, eval_every, iterations, gamma_threshold, random_state, minimum_probability, minimum_phi_value, per_word_topics, dtype)\u001b[0m\n\u001b[1;32m 183\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(alpha, \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;129;01mand\u001b[39;00m alpha \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mauto\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m 184\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mauto-tuning alpha not implemented in LdaMulticore; use plain LdaModel.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 186\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mLdaMulticore\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[1;32m 187\u001b[0m \u001b[43m \u001b[49m\u001b[43mcorpus\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcorpus\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_topics\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnum_topics\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 188\u001b[0m \u001b[43m \u001b[49m\u001b[43mid2word\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mid2word\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mchunksize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunksize\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpasses\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpasses\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43malpha\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43malpha\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meta\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43meta\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 189\u001b[0m \u001b[43m \u001b[49m\u001b[43mdecay\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecay\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moffset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moffset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meval_every\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43meval_every\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43miterations\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43miterations\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 190\u001b[0m \u001b[43m \u001b[49m\u001b[43mgamma_threshold\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgamma_threshold\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrandom_state\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrandom_state\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mminimum_probability\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mminimum_probability\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 191\u001b[0m \u001b[43m \u001b[49m\u001b[43mminimum_phi_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mminimum_phi_value\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mper_word_topics\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mper_word_topics\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 192\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/models/ldamodel.py:521\u001b[0m, in \u001b[0;36mLdaModel.__init__\u001b[0;34m(self, corpus, num_topics, id2word, distributed, chunksize, passes, update_every, alpha, eta, decay, offset, eval_every, iterations, gamma_threshold, minimum_probability, random_state, ns_conf, minimum_phi_value, per_word_topics, callbacks, dtype)\u001b[0m\n\u001b[1;32m 519\u001b[0m use_numpy \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdispatcher \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 520\u001b[0m start \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mtime()\n\u001b[0;32m--> 521\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mupdate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcorpus\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mchunks_as_numpy\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_numpy\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 522\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39madd_lifecycle_event(\n\u001b[1;32m 523\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcreated\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 524\u001b[0m msg\u001b[38;5;241m=\u001b[39m\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrained \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m in \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtime\u001b[38;5;241m.\u001b[39mtime()\u001b[38;5;250m \u001b[39m\u001b[38;5;241m-\u001b[39m\u001b[38;5;250m \u001b[39mstart\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.2f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124ms\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 525\u001b[0m )\n", "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/site-packages/gensim/models/ldamulticore.py:286\u001b[0m, in \u001b[0;36mLdaMulticore.update\u001b[0;34m(self, corpus, chunks_as_numpy)\u001b[0m\n\u001b[1;32m 283\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlog_perplexity(chunk, total_docs\u001b[38;5;241m=\u001b[39mlencorpus)\n\u001b[1;32m 285\u001b[0m logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtraining LDA model using \u001b[39m\u001b[38;5;132;01m%i\u001b[39;00m\u001b[38;5;124m processes\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mworkers)\n\u001b[0;32m--> 286\u001b[0m pool \u001b[38;5;241m=\u001b[39m \u001b[43mPool\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mworkers\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mworker_e_step\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mjob_queue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresult_queue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 287\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m pass_ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpasses):\n\u001b[1;32m 288\u001b[0m queue_size, reallen \u001b[38;5;241m=\u001b[39m [\u001b[38;5;241m0\u001b[39m], \u001b[38;5;241m0\u001b[39m\n", "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/context.py:119\u001b[0m, in \u001b[0;36mBaseContext.Pool\u001b[0;34m(self, processes, initializer, initargs, maxtasksperchild)\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m'''Returns a process pool object'''\u001b[39;00m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpool\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Pool\n\u001b[0;32m--> 119\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mPool\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocesses\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minitializer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minitargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmaxtasksperchild\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 120\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_context\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/pool.py:215\u001b[0m, in \u001b[0;36mPool.__init__\u001b[0;34m(self, processes, initializer, initargs, maxtasksperchild, context)\u001b[0m\n\u001b[1;32m 213\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_processes \u001b[38;5;241m=\u001b[39m processes\n\u001b[1;32m 214\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 215\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_repopulate_pool\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 216\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:\n\u001b[1;32m 217\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m p \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pool:\n", "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/pool.py:306\u001b[0m, in \u001b[0;36mPool._repopulate_pool\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 305\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_repopulate_pool\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 306\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_repopulate_pool_static\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_ctx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mProcess\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 307\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_processes\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 308\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_pool\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_inqueue\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 309\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_outqueue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_initializer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 310\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_initargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 311\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_maxtasksperchild\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 312\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_wrap_exception\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/pool.py:329\u001b[0m, in \u001b[0;36mPool._repopulate_pool_static\u001b[0;34m(ctx, Process, processes, pool, inqueue, outqueue, initializer, initargs, maxtasksperchild, wrap_exception)\u001b[0m\n\u001b[1;32m 327\u001b[0m w\u001b[38;5;241m.\u001b[39mname \u001b[38;5;241m=\u001b[39m w\u001b[38;5;241m.\u001b[39mname\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mProcess\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mPoolWorker\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 328\u001b[0m w\u001b[38;5;241m.\u001b[39mdaemon \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m--> 329\u001b[0m \u001b[43mw\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstart\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 330\u001b[0m pool\u001b[38;5;241m.\u001b[39mappend(w)\n\u001b[1;32m 331\u001b[0m util\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124madded worker\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/process.py:121\u001b[0m, in \u001b[0;36mBaseProcess.start\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m _current_process\u001b[38;5;241m.\u001b[39m_config\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdaemon\u001b[39m\u001b[38;5;124m'\u001b[39m), \\\n\u001b[1;32m 119\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdaemonic processes are not allowed to have children\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 120\u001b[0m _cleanup()\n\u001b[0;32m--> 121\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_popen \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_Popen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 122\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sentinel \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_popen\u001b[38;5;241m.\u001b[39msentinel\n\u001b[1;32m 123\u001b[0m \u001b[38;5;66;03m# Avoid a refcycle if the target function holds an indirect\u001b[39;00m\n\u001b[1;32m 124\u001b[0m \u001b[38;5;66;03m# reference to the process object (see bpo-30775)\u001b[39;00m\n", "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/context.py:289\u001b[0m, in \u001b[0;36mSpawnProcess._Popen\u001b[0;34m(process_obj)\u001b[0m\n\u001b[1;32m 286\u001b[0m \u001b[38;5;129m@staticmethod\u001b[39m\n\u001b[1;32m 287\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_Popen\u001b[39m(process_obj):\n\u001b[1;32m 288\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpopen_spawn_posix\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Popen\n\u001b[0;32m--> 289\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mPopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocess_obj\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/popen_spawn_posix.py:32\u001b[0m, in \u001b[0;36mPopen.__init__\u001b[0;34m(self, process_obj)\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, process_obj):\n\u001b[1;32m 31\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_fds \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m---> 32\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mprocess_obj\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/popen_fork.py:19\u001b[0m, in \u001b[0;36mPopen.__init__\u001b[0;34m(self, process_obj)\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreturncode \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 18\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfinalizer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m---> 19\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_launch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocess_obj\u001b[49m\u001b[43m)\u001b[49m\n", "File \u001b[0;32m~/anaconda3/envs/maritime/lib/python3.12/multiprocessing/popen_spawn_posix.py:62\u001b[0m, in \u001b[0;36mPopen._launch\u001b[0;34m(self, process_obj)\u001b[0m\n\u001b[1;32m 60\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msentinel \u001b[38;5;241m=\u001b[39m parent_r\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(parent_w, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwb\u001b[39m\u001b[38;5;124m'\u001b[39m, closefd\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m---> 62\u001b[0m \u001b[43mf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetbuffer\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 63\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 64\u001b[0m fds_to_close \u001b[38;5;241m=\u001b[39m []\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "%%time\n", "\n", "import datetime\n", "import numpy as np\n", "from gensim.models import CoherenceModel\n", "\n", "print(datetime.datetime.now())\n", "\n", "for a in alpha:\n", " for b in beta:\n", " for num in num_topics:\n", " cv, pv = compute_coherence_values(corpus=docs_vecs, dictionary=doc_dict,k=num, a=a, b=b) \n", "\n", " model_topics.append(num) \n", " coherence_values.append(cv) \n", " perplexity_values.append(pv)\n", " alpha_result.append(a)\n", " beta_result.append(b)\n", " print(\"#Topics: \" + str(num) + \", CV Score: \" + str(coherence_values[-1]) + \", PV Score: \" + str(perplexity_values[-1]) + \", Alpha: \" + str(alpha_result[-1]) + \", Beta: \" + str(beta_result[-1]))\n", " \n", "print(datetime.datetime.now())" ] }, { "cell_type": "markdown", "id": "364ff6d5-e3da-4dde-a2c8-5375fc5d711f", "metadata": {}, "source": [ "The table below reveals the top 20 fine tuned models with best combinations of coherence score and perplexity score. It was sorted by the coherence score in descending order as a higher coherence score indicates a better model, and sorted the perplexity score in ascending order as a lower perplexity score indicates a better model. While coherence score evaluates the quality of the topics, the perplexity score evaluates the overall performance of the model in predicting new documents. Usually, the coherence score is a better metric to use if the goal is to obtain topics that are semantically coherent and interpretable. Perplexity score, on the other hand, is a better metric to use if the goal is to build a model that generalises well to new data, in other words, how confident the model is in predicting the new data (Sánchez-Aguayo, et al., 2022). Ultimately, we aim to get a balance between the perplexity value and coherence score when determining our final model." ] }, { "cell_type": "code", "execution_count": null, "id": "78a60032-a4d7-44d4-841c-a1bd3740d5dd", "metadata": {}, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "# Find the top 20 combinations based on Coherence Score and Perplexity Score\n", "result = pd.DataFrame(\n", " {'Topics': model_topics,\n", " 'Coherence Score': coherence_values,\n", " 'Perplexity Score': perplexity_values,\n", " 'Alpha': alpha_result,\n", " 'Beta': beta_result\n", " })\n", "result.sort_values(by=['Coherence Score', 'Perplexity Score'], ascending=[False, True]).head(20)" ] }, { "cell_type": "code", "execution_count": null, "id": "3461df57-c069-4ad2-80d7-8890dec9438e", "metadata": {}, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "result.to_csv('data/lda_fine_tuning_result_moderate.csv')" ] }, { "cell_type": "code", "execution_count": null, "id": "800e5a4b-7302-42e8-97b0-5b598c1c80ae", "metadata": { "scrolled": true }, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "# Show graph Topics vs Coherence Score\n", "result.groupby('Alpha').plot(x='Topics', y='Coherence Score', legend = True)" ] }, { "cell_type": "code", "execution_count": null, "id": "26996b89-0e7a-4f2d-8cf7-c4a716569bc2", "metadata": {}, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "# Show graph Topics vs Perplexity Score\n", "\n", "plt.plot(model_topics, coherence_values)\n", "plt.xlabel(\"Num Topics\")\n", "plt.ylabel(\"Coherence Score\")\n", "plt.legend((\"Coherence Score\"), loc='best')\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "id": "91d2f4c1-de77-44b6-b41b-fcc9a07233e8", "metadata": {}, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "# Show graph Topics vs Perplexity Score\n", "\n", "plt.plot(model_topics, perplexity_values)\n", "plt.xlabel(\"Num Topics\")\n", "plt.ylabel(\"Perplexity score\")\n", "plt.legend((\"perplexity_values\"), loc='best')\n", "plt.show()" ] }, { "cell_type": "markdown", "id": "cdc3ddd2-f743-4e5b-b6c6-2656e0b77aec", "metadata": {}, "source": [ "## Final Model" ] }, { "cell_type": "code", "execution_count": null, "id": "490734ed-077c-4fb0-930c-0b42f4f63c94", "metadata": {}, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "# realised that there may be some overlaps for more than 5 topics, but below 5 topics results in low differentiation and high ambiguity among the topics.\n", "# LDA is not suitable for this dataset\n", "k = 9\n", "a = 'symmetric'\n", "# a = 0.31\n", "# b = 0.31\n", "b = 'symmetric'\n", "\n", "\n", "final_model = gensim.models.LdaMulticore(corpus=docs_vecs,\n", " id2word=doc_dict,\n", " num_topics=k, \n", " random_state=42,\n", " chunksize=100,\n", " passes=10,\n", " alpha=a,\n", " eta=b)" ] }, { "cell_type": "code", "execution_count": null, "id": "afe8abf0-2d12-414e-92be-a655865addb1", "metadata": { "tags": [] }, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "compute_coherence_values(corpus=docs_vecs, dictionary=doc_dict,k=k, a=a, b=b) " ] }, { "cell_type": "code", "execution_count": null, "id": "8430a827-6dbb-4737-8ccc-78ed17a01234", "metadata": { "tags": [] }, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "#Set up the environment to display the graphical outputs\n", "# feed the LDA model into the pyLDAvis instance\n", "pyLDAvis.enable_notebook()\n", "visual= gensimvis.prepare(final_model, docs_vecs, doc_dict)\n", "\n", "#Save the output to the html file\n", "pyLDAvis.save_html(visual, \"data/topic_viz12_mod_training.html\")" ] }, { "cell_type": "code", "execution_count": null, "id": "5e30d71a-a3c7-40c7-94c0-7eea1bedc887", "metadata": { "tags": [] }, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "final_model.print_topics(num_words=30)" ] }, { "cell_type": "markdown", "id": "607d2cfd-b3ca-4f99-9e01-d320ca98a2a0", "metadata": {}, "source": [ "# Save the final model " ] }, { "cell_type": "code", "execution_count": null, "id": "84eb2746-173a-4283-bca5-681f77548698", "metadata": {}, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "# Save a model to disk, or reload a pre-trained model\n", "# naming convention: final_model_topic_alpha_eta\n", "final_model.save(\"models/final_model_9_sym_sym\")" ] }, { "cell_type": "markdown", "id": "a7b6e4d9-a577-4dfb-ba6e-fc74365880f4", "metadata": {}, "source": [ "# Find dominant topic(s) for each news article" ] }, { "cell_type": "markdown", "id": "0eeecbcb-358c-44f9-8463-75cdfac0ba90", "metadata": {}, "source": [ "Attach the dominant topics back to the news dataset for classifying purpose." ] }, { "cell_type": "markdown", "id": "8bebb269-dbb0-4c46-925c-38de0f2bcfd7", "metadata": {}, "source": [ "Made use of gensim lda's own function: https://radimrehurek.com/gensim/models/ldamodel.html" ] }, { "cell_type": "code", "execution_count": null, "id": "f585ff52-b60d-4d70-ae64-a7c23d2cc6c1", "metadata": {}, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "import warnings\n", "warnings.filterwarnings('ignore')\n", "\n", "def format_topics_sentences(ldamodel, corpus, data):\n", " # Preallocate memory for the DataFrame\n", " num_docs = len(corpus)\n", " sent_topics = {'Dominant_Topic': [0] * num_docs, 'Perc_Contribution': [0.0] * num_docs, 'Topic_Distribution': [()] * num_docs}\n", " \n", " # Get main topic in each document\n", " for i, row in enumerate(ldamodel[corpus]):\n", " row = sorted(row, key=lambda x: (x[1]), reverse=True)\n", " if row:\n", " # Get the Dominant topic, Perc Contribution and Keywords for each document\n", " dominant_topic, perc_contribution = row[0]\n", " topic_distribution = row\n", " sent_topics['Dominant_Topic'][i] = int(dominant_topic)\n", " sent_topics['Perc_Contribution'][i] = round(perc_contribution, 4)\n", " sent_topics['Topic_Distribution'][i] = topic_distribution\n", "\n", " # Create the DataFrame\n", " sent_topics_df = pd.DataFrame(sent_topics)\n", " sent_topics_df['Text'] = data\n", "\n", " return sent_topics_df" ] }, { "cell_type": "code", "execution_count": null, "id": "24d3ff60-035e-4133-9ffd-88cce5cdccb1", "metadata": {}, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "df_topic_sents_keywords = format_topics_sentences(ldamodel=final_model, corpus=docs_vecs, data=cleaned.Headline_Details)" ] }, { "cell_type": "code", "execution_count": null, "id": "c88b088b", "metadata": {}, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "# Format\n", "df_dominant_topic = df_topic_sents_keywords.reset_index()\n", "df_dominant_topic.columns = ['Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'Topic_Distribution', 'Text']\n", "\n", "# Show\n", "df_dominant_topic.head(10)" ] }, { "cell_type": "markdown", "id": "560da382-aa86-4df1-8b85-56b057a27cd4", "metadata": {}, "source": [ "# Result Analysis" ] }, { "cell_type": "code", "execution_count": null, "id": "4fe6b40b-6922-4de3-8d9e-dac7474b6303", "metadata": {}, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "df_dominant_topic[\"Dominant_Topic\"].value_counts()" ] }, { "cell_type": "code", "execution_count": null, "id": "b9917340-31cf-48af-871f-b481128fdf22", "metadata": {}, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "import matplotlib.pyplot as plt\n", "\n", "# Get value counts of each topic\n", "topic_counts = df_dominant_topic[\"Dominant_Topic\"].value_counts()\n", "\n", "# Create a bar plot\n", "plt.figure(figsize=(8, 6))\n", "topic_counts.plot(kind=\"bar\", color=\"skyblue\")\n", "\n", "# Add labels to the bars\n", "for i, count in enumerate(topic_counts):\n", " plt.text(i, count, str(count), ha=\"center\", va=\"bottom\")\n", "\n", "# Add labels and title\n", "plt.xlabel(\"Topics\")\n", "plt.ylabel(\"Number of News\")\n", "plt.title(\"Topic Distribution\")\n", "\n", "# Show the plot\n", "plt.xticks(rotation=45) # Rotate x-axis labels for better readability\n", "plt.tight_layout()\n", "plt.show()\n" ] }, { "cell_type": "code", "execution_count": null, "id": "70e7d652-4421-45e0-93f8-aaa51c186422", "metadata": {}, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "df_dominant_topic.shape" ] }, { "cell_type": "code", "execution_count": null, "id": "69932a6e-7159-46b1-98f8-827d99b95c54", "metadata": {}, "outputs": [ { "ename": "", "evalue": "", "output_type": "error", "traceback": [ "\u001b[1;31mFailed to start the Kernel. \n", "\u001b[1;31mUnable to start Kernel 'maritime (Python 3.12.4)' due to a timeout waiting for the ports to get used. \n", "\u001b[1;31mView Jupyter log for further details." ] } ], "source": [ "# Sample 100 rows, can change the random_state for different samples\n", "sampled_data = df_dominant_topic.sample(n=100, random_state=42) \n", "sampled_df = pd.DataFrame(sampled_data).reset_index()\n", "sampled_df.to_csv('data/sample_moderate.csv')" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.4" } }, "nbformat": 4, "nbformat_minor": 5 }