fixed 2020 and state data

get_party.ipynb

@@ -41,7 +41,7 @@
     "#getting party\n",
     "state = (conn\n",
     "        .read_csv(\"1976-2020-president.csv\")\n",
-    "        .filter(_. year >=2000)\n",
+    "        # .filter(_. year >=2000)\n",
     "        .rename(state=\"state_po\" , party = \"party_simplified\")  # rename columns\n",
     "        .group_by([\"year\", \"state\"])\n",
     "        .aggregate(party=_.party.argmax(_.candidatevotes))  # winning party \n",
@@ -90,11 +90,17 @@
     "# getting party\n",
     "county = (conn\n",
     "        .read_csv(\"countypres_2000-2020.csv\")\n",
-    "        .filter((_.mode == \"TOTAL\") & (_.totalvotes > 0))  # filter only final votes\n",
+    "        .filter((_.totalvotes > 0))  # filter empty votes\n",
     "        .rename(state=\"state_po\", state_name = \"state\") \n",
     "        .mutate(county = _.county_name + ibis.literal(\" COUNTY\"))\n",
-    "        .group_by([\"year\", \"state\", \"county\",\"state_name\"])\n",
-    "        .aggregate(party=_.party.argmax(_.candidatevotes))  # winning party \n",
+    "        .group_by([\"year\", \"state\", \"county\", \"state_name\", \"party\"])\n",
+    "        .aggregate(\n",
+    "            total_candidate_votes=_.candidatevotes.sum()  #getting total votes per candidate \n",
+    "        )\n",
+    "        .group_by([\"year\", \"state\", \"county\", \"state_name\"])\n",
+    "        .aggregate(\n",
+    "            party=_.party.argmax(_.total_candidate_votes)  # party with the highest total votes\n",
+    "        )\n",
     "        .select(\"year\", \"state\", \"county\", \"party\",\"state_name\")  \n",
     ")"
    ]
@@ -117,7 +123,6 @@
     "        .cast({\"geometry\": \"geometry\",\"municipal\": \"string\"})\n",
     "        .mutate(jurisdiction = ibis.literal(\"County\"))\n",
     "        .select(\"state\", \"county\", \"municipal\",\"jurisdiction\",\"geometry\", \"year\", \"party\")\n",
-    "\n",
     "        )"
    ]
   },
@@ -225,7 +230,7 @@
     "gdf_city = df_city.execute().set_crs(\"EPSG:4326\")\n",
     "gdf_city.to_file(\"party_municipal.geojson\")\n",
     "\n",
-    "generate_pmtiles(\"party_state.geojson\", \"party_county.geojson\",\"party_municipal.geojson\", \"landvote_polygons.pmtiles\")\n",
+    "generate_pmtiles(\"party_state.geojson\", \"party_county.geojson\",\"party_municipal.geojson\", \"party_polygons.pmtiles\")\n",
     "hf_upload(\"party_polygons.pmtiles\", \"boettiger-lab/landvote\")\n"
    ]
   },
@@ -249,14 +254,6 @@
     "df.execute().set_crs(\"EPSG:4326\").to_parquet(\"party_polygons.parquet\")\n",
     "hf_upload(\"party_polygons.parquet\", \"boettiger-lab/landvote\")\n"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "3e87fc40-b535-4a44-9230-99f6777e7c48",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
@@ -275,7 +272,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.10"
+   "version": "3.10.12"
   }
  },
  "nbformat": 4,

preprocess.ipynb

@@ -10,7 +10,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "6452373c-d10f-472c-9174-cd05a3363587",
    "metadata": {},
    "outputs": [],
@@ -30,8 +30,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "371ef7f4-95b9-49f0-80ba-48807b98b052",
+   "execution_count": 2,
+   "id": "dd9ffc64-b4cf-4e5b-9c96-703e91a77837",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -43,16 +43,18 @@
     "            .mutate(elect_year = _.year - _.year % 4) # get most recent election year \n",
     "            .cast({\"municipal\": \"string\",\"county\":\"string\"})\n",
     "            .mutate(municipal=ibis.case()\n",
-    "                    .when(_.jurisdiction.isin(['State','County']), ibis.literal(\"-\")) \n",
-    "                    .else_(_.municipal) \n",
-    "                    .end()\n",
-    "                    )\n",
-    "                    .mutate(county=ibis.case()\n",
-    "                    .when(_.jurisdiction.isin(['State']),ibis.literal(\"-\")) \n",
-    "                    .else_(_.county) \n",
-    "                    .end()\n",
-    "                    )\n",
-    "            )\n",
+    "                 .when(_.jurisdiction.isin(['State','County']), ibis.literal(\"-\")) \n",
+    "                 .else_(_.municipal) \n",
+    "                 .end()\n",
+    "                 )\n",
+    "            .mutate(county=ibis.case()\n",
+    "                 .when(_.jurisdiction.isin(['State']), ibis.literal(\"-\"))\n",
+    "                 .else_(ibis.case()\n",
+    "                         .when(_.county.endswith('COUNTY'), _.county)\n",
+    "                         .else_(_.county + ' COUNTY')\n",
+    "                         .end())\n",
+    "                 .end())\n",
+    "           )\n",
     "\n",
     "party = (conn\n",
     "        .read_parquet(party_url)\n",
@@ -72,13 +74,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "8fc40e96-fffd-4b23-9963-c931fdce96f6",
    "metadata": {},
    "outputs": [],
    "source": [
     "votes = (landvote\n",
-    "        .join(party,[\"state\",\"county\",\"municipal\",\"jurisdiction\",'geometry', _.elect_year == party[\"year\"]],how = \"inner\")\n",
+    "        .join(party,[\"state\",\"county\",\"municipal\",\"jurisdiction\",\"geometry\", _.elect_year == party[\"year\"]],how = \"inner\")\n",
     "        .drop('elect_year','year_right')\n",
     "        .mutate(municipal=ibis.case()\n",
     "                .when(_.municipal == ibis.literal(\"-\"), None) \n",
@@ -93,14 +95,6 @@
     "        )"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "65e75bc0-fef0-48f2-a543-7aae999579bf",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "markdown",
    "id": "e80cfd2e-40eb-4065-9ae6-dcaf83319d9a",
@@ -111,10 +105,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "b1cd8f44-57fa-49a8-b438-f9e4aab747c5",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n",
+      "Token is valid (permission: write).\n",
+      "Your token has been saved to /home/rstudio/.cache/huggingface/token\n",
+      "Login successful\n"
+     ]
+    }
+   ],
    "source": [
     "import subprocess\n",
     "import os\n",
@@ -160,10 +165,85 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "7061577e-0632-4395-8ac5-241a1fab53b0",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "341b737abd4c4fceae622965133c09a2",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "859a3d133ac1491f8841d61bc2da25bf",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b0e18752ba4143bf88c8b447e521e8a6",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "1576 features, 14118861 bytes of geometry and attributes, 45035 bytes of string pool, 0 bytes of vertices, 0 bytes of nodes\n",
+      "Choosing a maxzoom of -z3 for features typically 91177 feet (27791 meters) apart, and at least 11358 feet (3462 meters) apart\n",
+      "Choosing a maxzoom of -z10 for resolution of about 308 feet (93 meters) within features\n",
+      "  99.9%  10/261/364  \n",
+      "  100.0%  10/55/448  \r"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Successfully generated PMTiles file: votes.pmtiles\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "17feadfbbbc54429b56aa85a8087310b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "votes.pmtiles:   0%|          | 0.00/4.08M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "gdf_state = votes.filter(_.jurisdiction == 'State').execute().set_crs(\"EPSG:4326\")\n",
     "gdf_state.to_file(\"votes_state.geojson\")\n",
@@ -180,10 +260,39 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "f2979624-bcdf-4a8a-899a-c22fc3cdaf0e",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "cc5d666af6bd4f9da53710f207f6103b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9a0c8f8fb5bd4486af8220e633cd9c97",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "votes.parquet:   0%|          | 0.00/31.7M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "# save as parquet\n",
     "votes.execute().set_crs(\"EPSG:4326\").to_parquet(\"votes.parquet\")\n",
@@ -215,7 +324,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.10"
+   "version": "3.10.12"
   }
  },
  "nbformat": 4,