adding municipals

get_party.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "id": "d179ded1-6235-47ed-bbfb-6d72468188d5",
    "metadata": {},
    "outputs": [],
@@ -20,12 +20,41 @@
     "votes = conn.read_parquet(\"vote.parquet\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "170ba045-8848-4a99-a4f6-68bde22428af",
+   "metadata": {},
+   "source": [
+    "# Getting party affiliations for counties"
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "id": "ab644102-c725-4cf4-915c-8550a0a74c32",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "       year                    key current_party previous_party\n",
+      "0      2004      AUTAUGA COUNTY-AL    REPUBLICAN     REPUBLICAN\n",
+      "1      2004    COVINGTON COUNTY-AL    REPUBLICAN     REPUBLICAN\n",
+      "2      2004       ELMORE COUNTY-AL    REPUBLICAN     REPUBLICAN\n",
+      "3      2004        HENRY COUNTY-AL    REPUBLICAN     REPUBLICAN\n",
+      "4      2004      LOWNDES COUNTY-AL      DEMOCRAT       DEMOCRAT\n",
+      "...     ...                    ...           ...            ...\n",
+      "14904  2020          ELK COUNTY-PA    REPUBLICAN     REPUBLICAN\n",
+      "14905  2020       LOVING COUNTY-TX    REPUBLICAN     REPUBLICAN\n",
+      "14906  2020    RED RIVER COUNTY-TX    REPUBLICAN     REPUBLICAN\n",
+      "14907  2020      WHATCOM COUNTY-WA      DEMOCRAT       DEMOCRAT\n",
+      "14908  2020  DISTRICT 12 COUNTY-AK    REPUBLICAN     REPUBLICAN\n",
+      "\n",
+      "[14909 rows x 4 columns]\n"
+     ]
+    }
+   ],
    "source": [
     "filtered = county.filter((_.mode == \"TOTAL\") & (_.totalvotes > 0))\n",
     "\n",
@@ -73,14 +102,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 13,
    "id": "ce0d80bf-3b78-4aa9-8048-5cc0dbf970d9",
    "metadata": {},
    "outputs": [],
    "source": [
     "df = (votes\n",
     "        .mutate(key = _.key.upper())\n",
-    "        .filter(_.jurisdiction == \"County\")\n",
+    "        .filter(_.jurisdiction == \"Municipal\")\n",
     "        .join(county_parties, [\"key\",\"year\"],how='inner'\n",
     "        )\n",
     "        .cast({\"geometry\": \"geometry\"})\n",
@@ -89,10 +118,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 14,
    "id": "87bef5e2-a40a-4aff-aa27-e7d49ec68aac",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n",
+      "Token is valid (permission: write).\n",
+      "Your token has been saved to /home/rstudio/.cache/huggingface/token\n",
+      "Login successful\n"
+     ]
+    }
+   ],
    "source": [
     "import subprocess\n",
     "import os\n",
@@ -137,7 +177,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "b086e76c-4285-4036-8033-e4e45cb6966b",
    "metadata": {},
    "outputs": [],
@@ -145,24 +185,22 @@
     "gdf= df.execute()\n",
     "gdf = gdf.set_crs(\"EPSG:4326\")\n",
     "\n",
-    "gdf.to_parquet(\"county_parties.parquet\")\n",
-    "hf_upload(\"county_parties.parquet\", \"boettiger-lab/landvote\")\n",
+    "# gdf.to_parquet(\"county_parties.parquet\")\n",
+    "# hf_upload(\"county_parties.parquet\", \"boettiger-lab/landvote\")\n",
     "\n",
-    "gdf.to_file(\"county_parties.geojson\")\n",
-    "hf_upload(\"county_parties.geojson\", \"boettiger-lab/landvote\")\n",
+    "# gdf.to_file(\"county_parties.geojson\")\n",
+    "# hf_upload(\"county_parties.geojson\", \"boettiger-lab/landvote\")\n",
     "\n",
-    "generate_pmtiles(\"county_parties.geojson\", \"county_parties.pmtiles\")\n",
-    "hf_upload(\"county_parties.pmtiles\", \"boettiger-lab/landvote\")\n"
+    "# generate_pmtiles(\"county_parties.geojson\", \"county_parties.pmtiles\")\n",
+    "# hf_upload(\"county_parties.pmtiles\", \"boettiger-lab/landvote\")\n"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "cb790ed9-6cb8-4705-abaa-ce0008851a87",
+   "cell_type": "markdown",
+   "id": "6193c4b9-0183-4aae-9a25-899a748fd65e",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "gdf"
+    "# Checking map"
    ]
   },
   {
@@ -217,10 +255,311 @@
     "m\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "8c12fd16-a953-4273-9e0f-44b50eacf633",
+   "metadata": {},
+   "source": [
+    "# Getting Municipals "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "b3429fea-7c0d-4838-bcbb-6552079dc3b6",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>key</th>\n",
+       "      <th>geometry</th>\n",
+       "      <th>Status</th>\n",
+       "      <th>yes</th>\n",
+       "      <th>year</th>\n",
+       "      <th>amount</th>\n",
+       "      <th>log_amount</th>\n",
+       "      <th>jurisdiction</th>\n",
+       "      <th>current_party</th>\n",
+       "      <th>previous_party</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>San Luis Obispo-CA</td>\n",
+       "      <td>POLYGON ((-120.58962 35.28275, -120.59096 35.2...</td>\n",
+       "      <td>Pass</td>\n",
+       "      <td>58.23%</td>\n",
+       "      <td>2020</td>\n",
+       "      <td>$21,600,000</td>\n",
+       "      <td>16.8882</td>\n",
+       "      <td>Municipal</td>\n",
+       "      <td>DEMOCRAT</td>\n",
+       "      <td>DEMOCRAT</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Fruita-CO</td>\n",
+       "      <td>POLYGON ((-108.65899 39.15887, -108.66033 39.1...</td>\n",
+       "      <td>Pass</td>\n",
+       "      <td>60.73%</td>\n",
+       "      <td>2020</td>\n",
+       "      <td>$1,300,000</td>\n",
+       "      <td>14.0779</td>\n",
+       "      <td>Municipal</td>\n",
+       "      <td>REPUBLICAN</td>\n",
+       "      <td>REPUBLICAN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Gig Harbor-WA</td>\n",
+       "      <td>POLYGON ((-122.51787 47.32597, -122.51921 47.3...</td>\n",
+       "      <td>Pass</td>\n",
+       "      <td>62.05%</td>\n",
+       "      <td>2004</td>\n",
+       "      <td>$3,500,000</td>\n",
+       "      <td>15.0683</td>\n",
+       "      <td>Municipal</td>\n",
+       "      <td>DEMOCRAT</td>\n",
+       "      <td>DEMOCRAT</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Jupiter-FL</td>\n",
+       "      <td>POLYGON ((-80.02421 26.93422, -80.02555 26.920...</td>\n",
+       "      <td>Pass</td>\n",
+       "      <td>70.84%</td>\n",
+       "      <td>2004</td>\n",
+       "      <td>$17,000,000</td>\n",
+       "      <td>16.6487</td>\n",
+       "      <td>Municipal</td>\n",
+       "      <td>DEMOCRAT</td>\n",
+       "      <td>DEMOCRAT</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>Bozeman-MT</td>\n",
+       "      <td>POLYGON ((-110.97405 45.67943, -110.97539 45.6...</td>\n",
+       "      <td>Pass</td>\n",
+       "      <td>73.39%</td>\n",
+       "      <td>2012</td>\n",
+       "      <td>$7,500,000</td>\n",
+       "      <td>15.8304</td>\n",
+       "      <td>Municipal</td>\n",
+       "      <td>REPUBLICAN</td>\n",
+       "      <td>DEMOCRAT</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>268</th>\n",
+       "      <td>Chandler-AZ</td>\n",
+       "      <td>POLYGON ((-111.77118 33.3062, -111.77253 33.29...</td>\n",
+       "      <td>Pass</td>\n",
+       "      <td>75.16%</td>\n",
+       "      <td>2004</td>\n",
+       "      <td>$17,350,000</td>\n",
+       "      <td>16.6691</td>\n",
+       "      <td>Municipal</td>\n",
+       "      <td>REPUBLICAN</td>\n",
+       "      <td>REPUBLICAN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>269</th>\n",
+       "      <td>Medfield-MA</td>\n",
+       "      <td>POLYGON ((-71.23546 42.18656, -71.23681 42.172...</td>\n",
+       "      <td>Pass</td>\n",
+       "      <td>84.17%</td>\n",
+       "      <td>2008</td>\n",
+       "      <td>$1,550,000</td>\n",
+       "      <td>14.2538</td>\n",
+       "      <td>Municipal</td>\n",
+       "      <td>DEMOCRAT</td>\n",
+       "      <td>DEMOCRAT</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>270</th>\n",
+       "      <td>Somerville-MA</td>\n",
+       "      <td>POLYGON ((-71.0295 42.3876, -71.03084 42.37394...</td>\n",
+       "      <td>Pass</td>\n",
+       "      <td>76.07%</td>\n",
+       "      <td>2012</td>\n",
+       "      <td>$9,659,333</td>\n",
+       "      <td>16.0834</td>\n",
+       "      <td>Municipal</td>\n",
+       "      <td>DEMOCRAT</td>\n",
+       "      <td>DEMOCRAT</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>271</th>\n",
+       "      <td>Dryden-NY</td>\n",
+       "      <td>POLYGON ((-76.22716 42.49091, -76.2285 42.4772...</td>\n",
+       "      <td>Fail</td>\n",
+       "      <td>44.07%</td>\n",
+       "      <td>2016</td>\n",
+       "      <td>$71,800</td>\n",
+       "      <td>11.1816</td>\n",
+       "      <td>Municipal</td>\n",
+       "      <td>DEMOCRAT</td>\n",
+       "      <td>DEMOCRAT</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>272</th>\n",
+       "      <td>Grafton-WI</td>\n",
+       "      <td>POLYGON ((-87.88178 43.31952, -87.88313 43.305...</td>\n",
+       "      <td>Fail</td>\n",
+       "      <td>28.78%</td>\n",
+       "      <td>2004</td>\n",
+       "      <td>$1,120,000</td>\n",
+       "      <td>13.9288</td>\n",
+       "      <td>Municipal</td>\n",
+       "      <td>REPUBLICAN</td>\n",
+       "      <td>REPUBLICAN</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>273 rows × 10 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                    key                                           geometry  \\\n",
+       "0    San Luis Obispo-CA  POLYGON ((-120.58962 35.28275, -120.59096 35.2...   \n",
+       "1             Fruita-CO  POLYGON ((-108.65899 39.15887, -108.66033 39.1...   \n",
+       "2         Gig Harbor-WA  POLYGON ((-122.51787 47.32597, -122.51921 47.3...   \n",
+       "3            Jupiter-FL  POLYGON ((-80.02421 26.93422, -80.02555 26.920...   \n",
+       "4            Bozeman-MT  POLYGON ((-110.97405 45.67943, -110.97539 45.6...   \n",
+       "..                  ...                                                ...   \n",
+       "268         Chandler-AZ  POLYGON ((-111.77118 33.3062, -111.77253 33.29...   \n",
+       "269         Medfield-MA  POLYGON ((-71.23546 42.18656, -71.23681 42.172...   \n",
+       "270       Somerville-MA  POLYGON ((-71.0295 42.3876, -71.03084 42.37394...   \n",
+       "271           Dryden-NY  POLYGON ((-76.22716 42.49091, -76.2285 42.4772...   \n",
+       "272          Grafton-WI  POLYGON ((-87.88178 43.31952, -87.88313 43.305...   \n",
+       "\n",
+       "    Status     yes  year       amount  log_amount jurisdiction current_party  \\\n",
+       "0     Pass  58.23%  2020  $21,600,000     16.8882    Municipal      DEMOCRAT   \n",
+       "1     Pass  60.73%  2020   $1,300,000     14.0779    Municipal    REPUBLICAN   \n",
+       "2     Pass  62.05%  2004   $3,500,000     15.0683    Municipal      DEMOCRAT   \n",
+       "3     Pass  70.84%  2004  $17,000,000     16.6487    Municipal      DEMOCRAT   \n",
+       "4     Pass  73.39%  2012   $7,500,000     15.8304    Municipal    REPUBLICAN   \n",
+       "..     ...     ...   ...          ...         ...          ...           ...   \n",
+       "268   Pass  75.16%  2004  $17,350,000     16.6691    Municipal    REPUBLICAN   \n",
+       "269   Pass  84.17%  2008   $1,550,000     14.2538    Municipal      DEMOCRAT   \n",
+       "270   Pass  76.07%  2012   $9,659,333     16.0834    Municipal      DEMOCRAT   \n",
+       "271   Fail  44.07%  2016      $71,800     11.1816    Municipal      DEMOCRAT   \n",
+       "272   Fail  28.78%  2004   $1,120,000     13.9288    Municipal    REPUBLICAN   \n",
+       "\n",
+       "    previous_party  \n",
+       "0         DEMOCRAT  \n",
+       "1       REPUBLICAN  \n",
+       "2         DEMOCRAT  \n",
+       "3         DEMOCRAT  \n",
+       "4         DEMOCRAT  \n",
+       "..             ...  \n",
+       "268     REPUBLICAN  \n",
+       "269       DEMOCRAT  \n",
+       "270       DEMOCRAT  \n",
+       "271       DEMOCRAT  \n",
+       "272     REPUBLICAN  \n",
+       "\n",
+       "[273 rows x 10 columns]"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "\n",
+    "localities_boundaries = \"us_localities.parquet\"\n",
+    "locality = conn.read_parquet(localities_boundaries)\n",
+    "landvote = conn.read_csv(\"landvote.csv\")\n",
+    "\n",
+    "#needed to redo this, since I didn't save county in \"votes.parquet\". \n",
+    "vote_local = (landvote\n",
+    "                .filter(_[\"Jurisdiction Type\"] == \"Municipal\")\n",
+    "                .rename(city = \"Jurisdiction Name\", state_id = \"State\")\n",
+    "                .mutate(key = _.city + ibis.literal('-') + _.state_id)\n",
+    "                .rename(amount = 'Conservation Funds at Stake', yes = '% Yes')\n",
+    "                .mutate(amount_n=_.amount.replace('$', '').replace(',', '').cast('float'))\n",
+    "                .mutate(log_amount=_.amount_n.log())\n",
+    "                .mutate(year=_['Date'].year().cast('int32'))\n",
+    "                .mutate(\n",
+    "                    yes=ibis.case()\n",
+    "                        .when(_.yes.isin(['Pass', 'None','Fail']), None)  # Handle non-numeric cases\n",
+    "                        .when(_.yes.notnull(), (_.yes.replace('%', '').cast('float').round(2).cast(dt.float64)).cast(dt.string) + '%')  # Convert valid percentages and add %\n",
+    "                        .else_(None)  # Default to None for other cases\n",
+    "                        .end()\n",
+    "                )\n",
+    "              .mutate(log_amount = _.log_amount.round(4))\n",
+    "                .select('key', 'Status', 'yes', 'year', 'amount', 'log_amount', )\n",
+    "                )\n",
+    "\n",
+    "# getting the county parties for each municipal  \n",
+    "df_municipals = (locality \n",
+    "            .mutate(key_municipal = _.name + ibis.literal('-') + _.state_id) \n",
+    "            .mutate(key = (_.county + ibis.literal('-') + _.state_id).upper()) \n",
+    "            .select('key', 'geometry','key_municipal','name')\n",
+    "            .right_join(vote_local, [_.key_municipal == vote_local[\"key\"]])\n",
+    "            .mutate(jurisdiction = ibis.literal(\"Municipal\"))\n",
+    "            .cast({\"geometry\": \"geometry\"})\n",
+    "            .mutate(geometry = _.geometry.buffer(.07))\n",
+    "            .join(county_parties, [\"key\",\"year\"],how='inner')\n",
+    "            .rename(county = \"key\")\n",
+    "            .rename(key = \"key_municipal\")\n",
+    "            .select('key','geometry','Status','yes','year','amount','log_amount','jurisdiction','current_party','previous_party')\n",
+    "           )\n",
+    "\n",
+    "\n",
+    "gdf_municipals = df_municipals.execute()\n",
+    "gdf_municipals = gdf_municipals.set_crs(\"EPSG:4326\")\n",
+    "gdf_municipals\n",
+    "\n",
+    "\n",
+    "# gdf_municipals.to_parquet(\"municipal_parties.parquet\")\n",
+    "# hf_upload(\"municipal_parties.parquet\", \"boettiger-lab/landvote\")\n",
+    "\n",
+    "# gdf_municipals.to_file(\"municipal_parties.geojson\")\n",
+    "# hf_upload(\"municipal_parties.geojson\", \"boettiger-lab/landvote\")\n",
+    "\n",
+    "# generate_pmtiles(\"municipal_parties.geojson\", \"municipal_parties.pmtiles\")\n",
+    "# hf_upload(\"municipal_parties.pmtiles\", \"boettiger-lab/landvote\")\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "e4280fb7-449a-4a1d-b760-67ce68fb5d92",
+   "id": "322b9a85-bdf9-45f9-9b19-695cc1b996e8",
    "metadata": {},
    "outputs": [],
    "source": []