added political parties

app.py

@@ -63,7 +63,8 @@ year = st.slider("Select a year", min_value=1988, max_value=2024, value=2022, st
 import leafmap.maplibregl as leafmap
 m = leafmap.Map(style="positron",   center=(-100, 40), zoom=3)
 
-url = "https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/landvote.pmtiles"
+url = "https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/landvote_polygons.pmtiles"
+# url = "https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/votes.pmtiles"
 
 dark_orange = 'rgba(171, 86, 1, 1)'   # dark orange - min value 
 light_orange = 'rgba(243, 211, 177, 1)' # light orange 
@@ -216,7 +217,7 @@ m.add_pmtiles(
     url,
     style=style_counties,
     visible=True,
-    opacity=0.6,
+    opacity=1.0,
     tooltip=True,
     fit_bounds=False
 )

get_city_polygons.ipynb

@@ -0,0 +1,151 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "4bfd881f-5889-4a26-b003-e2611708ad2a",
+   "metadata": {},
+   "source": [
+    "# Getting city polygons from Overture Maps"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3e5756d2-382b-49e9-93b5-2ecf6d0eb812",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import duckdb\n",
+    "\n",
+    "con = duckdb.connect()\n",
+    "\n",
+    "con.execute(\"SET s3_region='us-west-2';\")\n",
+    "con.execute(\"LOAD spatial;\")\n",
+    "con.execute(\"LOAD httpfs;\")\n",
+    "\n",
+    "# getting polygons of localities in the US.\n",
+    "query = \"\"\"\n",
+    "    COPY (\n",
+    "        SELECT * \n",
+    "        FROM read_parquet('s3://overturemaps-us-west-2/release/2024-09-18.0/theme=divisions/*/*')\n",
+    "        WHERE country = 'US' AND subtype IN ('locality')\n",
+    "    ) TO 'us_localities_raw.parquet' (FORMAT 'parquet');\n",
+    "\"\"\"\n",
+    "con.execute(query)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "25f62dd7-5539-438b-8f0a-1d85c9bc78ab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import ibis\n",
+    "from ibis import _\n",
+    "\n",
+    "conn = ibis.duckdb.connect(extensions=[\"spatial\"])\n",
+    "\n",
+    "df = (conn\n",
+    "      .read_parquet(\"us_localities_raw.parquet\")\n",
+    "      .cast({\"geometry\": \"geometry\"})\n",
+    "      .filter(_[\"type\"] == \"division\")\n",
+    "      .mutate(municipal = _.names[\"primary\"])\n",
+    "      .mutate(state = _.region.replace(\"US-\", \"\")) \n",
+    "      .mutate(county = _.hierarchies[0][2]['name'] ) #extract county from nested dictionary \n",
+    "      .mutate(key_long = _.municipal + ibis.literal('-') + _.county + ibis.literal('-') + _.state)\n",
+    "      .select(\"key_long\",\"municipal\", \"county\",\"state\" ,\"geometry\")\n",
+    "     )\n",
+    "\n",
+    "\n",
+    "## Dropping rows with same locality and state, with differing counties - landvote doesn't specify county for cities so we are dropping these to avoid duplicates. \n",
+    "county_count = (\n",
+    "    df.group_by([\"municipal\", \"state\"])\n",
+    "    .aggregate(county_count=_.county.nunique())  # Count unique counties for each group\n",
+    ") \n",
+    "valid_names = county_count.filter(county_count.county_count == 1).select(\"municipal\", \"state\")\n",
+    "df_filtered = df.join(valid_names, [\"municipal\", \"state\"], how=\"inner\")\n",
+    "\n",
+    "\n",
+    "# if two records have the same name but different geometries, only keep the first one. \n",
+    "df_unique = (\n",
+    "    df_filtered.group_by(\"key_long\")\n",
+    "    .aggregate(\n",
+    "        municipal=df_filtered.municipal.first(),\n",
+    "        county=df_filtered.county.first(),\n",
+    "        state=df_filtered.state.first(),\n",
+    "        geometry=df_filtered.geometry.first()\n",
+    "    )\n",
+    "    .mutate(geometry = _.geometry.buffer(.07))\n",
+    "    .select(\"state\",\"county\",\"municipal\",\"geometry\")\n",
+    ")\n",
+    "\n",
+    "df_unique.execute().to_parquet(\"us_localities.parquet\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0fce9fe8-584f-4260-9217-3aade9e71eef",
+   "metadata": {},
+   "source": [
+    "# Uploading city polygons to Hugging Face"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ca02743f-0bf4-46e5-91fd-a5fe37519ecd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import subprocess\n",
+    "import os\n",
+    "from huggingface_hub import HfApi, login\n",
+    "import streamlit as st\n",
+    "\n",
+    "login(st.secrets[\"HF_TOKEN\"])\n",
+    "api = HfApi()\n",
+    "\n",
+    "def hf_upload(file, repo_id):\n",
+    "    info = api.upload_file(\n",
+    "            path_or_fileobj=file,\n",
+    "            path_in_repo=file,\n",
+    "            repo_id=repo_id,\n",
+    "            repo_type=\"dataset\",\n",
+    "        )\n",
+    "hf_upload(\"us_localities.parquet\", \"boettiger-lab/landvote\")\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1ddadf0a-5b45-487f-a664-0c0696f75579",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

static-maps.ipynb → get_landvote.ipynb

@@ -1,5 +1,13 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "fa25e9ed-1dd1-4ca1-9898-9606c94d449f",
+   "metadata": {},
+   "source": [
+    "# Merging state, county, and city polygons with landvote data\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -10,7 +18,7 @@
     "import ibis\n",
     "from ibis import _\n",
     "import streamlit as st\n",
-    "import ibis.expr.datatypes as dt  # Make sure to import the necessary module\n",
+    "import ibis.expr.datatypes as dt  \n",
     "\n",
     "conn = ibis.duckdb.connect(extensions=[\"spatial\"])"
    ]
@@ -20,7 +28,7 @@
    "id": "943e4127-4af5-42c9-b1e2-48af2d888c24",
    "metadata": {},
    "source": [
-    "# Landvote data"
+    "# Landvote data\n"
    ]
   },
   {
@@ -78,16 +86,19 @@
    "outputs": [],
    "source": [
     "state_boundaries = \"https://data.source.coop/cboettig/us-boundaries/us-state-territory.parquet\"\n",
-    "states = conn.read_parquet(state_boundaries).rename(state_id = \"STUSPS\", state = \"NAME\")\n",
-    "\n",
     "landvote_states = landvote_df.filter(_.jurisdiction == \"State\")\n",
     "\n",
-    "df_state = (states\n",
-    "            .select('state_id','geometry')\n",
-    "            .cast({\"geometry\": \"geometry\"})\n",
-    "            .right_join(landvote_states, landvote_states['state'] == states['state_id'] )\n",
+    "state = (conn\n",
+    "        .read_parquet(state_boundaries)\n",
+    "        .rename(state = \"STUSPS\", state_name = \"NAME\")\n",
+    "        .select('state','geometry','state_name')\n",
+    "        .cast({\"geometry\": \"geometry\"})\n",
+    "        )\n",
+    "\n",
+    "df_state = (state\n",
+    "            .join(landvote_states, \"state\",how = \"inner\")\n",
     "            .select('state','county','municipal','jurisdiction','geometry','Status', 'yes', 'year', 'amount', 'log_amount', )\n",
-    "             )"
+    "            )"
    ]
   },
   {
@@ -108,13 +119,14 @@
     "landvote_county = landvote_df.filter(_.jurisdiction == \"County\")\n",
     "\n",
     "county_boundaries = \"https://data.source.coop/cboettig/us-boundaries/us-county.parquet\"\n",
-    "county = conn.read_parquet(county_boundaries).rename(county = \"NAMELSAD\", state = \"STATE_NAME\")\n",
     "\n",
-    "\n",
-    "df_county = (county\n",
-    "            .select('state','county','geometry')\n",
+    "df_county = (conn\n",
+    "            .read_parquet(county_boundaries)\n",
+    "            .rename(county = \"NAMELSAD\", state_name = \"STATE_NAME\")\n",
+    "            .join(state, \"state_name\", how = \"inner\")\n",
+    "            .select('state','state_name','county','geometry')\n",
     "            .cast({\"geometry\": \"geometry\"})\n",
-    "            .right_join(landvote_county, \"county\" )\n",
+    "            .join(landvote_county, [\"county\",\"state\"],how = \"inner\")\n",
     "            .select('state','county','municipal','jurisdiction','geometry','Status', 'yes', 'year', 'amount', 'log_amount', )\n",
     "             )"
    ]
@@ -134,20 +146,15 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "landvote_local = landvote_df.filter(_.jurisdiction == \"Municipal\")\n",
-    "\n",
-    "local = (conn\n",
-    "        .read_parquet(\"us_localities.parquet\")\n",
-    "        .rename(municipal = \"name\",state = \"state_id\")\n",
-    "        )\n",
+    "landvote_city = landvote_df.filter(_.jurisdiction == \"Municipal\")\n",
     "\n",
-    "df_local = (local\n",
+    "df_city = (conn\n",
+    "            .read_parquet(\"us_localities.parquet\")\n",
     "            .select('state','county','municipal','geometry')\n",
     "            .cast({\"geometry\": \"geometry\"})\n",
-    "            .mutate(geometry = _.geometry.buffer(.07))\n",
-    "            .right_join(landvote_local, [\"municipal\",\"state\"])\n",
+    "            .join(landvote_city, [\"municipal\",\"state\"], how = \"inner\")\n",
     "            .select('state','county','municipal','jurisdiction','geometry','Status', 'yes', 'year', 'amount', 'log_amount', )\n",
-    "             )"
+    "            )"
    ]
   },
   {
@@ -214,14 +221,17 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "df_state.execute().to_file(\"vote_state.geojson\")\n",
-    "df_county.execute().to_file(\"vote_county.geojson\")\n",
-    "df_local.execute().to_file(\"vote_municipal.geojson\")\n",
+    "gdf_state = df_state.execute().set_crs(\"EPSG:4326\")\n",
+    "gdf_state.to_file(\"landvote_state.geojson\")\n",
+    "\n",
+    "gdf_county = df_county.execute().set_crs(\"EPSG:4326\")\n",
+    "gdf_county.to_file(\"landvote_county.geojson\")\n",
     "\n",
-    "# hf_upload(\"vote.geojson\", \"boettiger-lab/landvote\")\n",
+    "gdf_city = df_city.execute().set_crs(\"EPSG:4326\")\n",
+    "gdf_city.to_file(\"landvote_municipal.geojson\")\n",
     "\n",
-    "generate_pmtiles(\"vote_state.geojson\", \"vote_county.geojson\",\"vote_municipal.geojson\", \"landvote.pmtiles\")\n",
-    "hf_upload(\"landvote.pmtiles\", \"boettiger-lab/landvote\")\n"
+    "generate_pmtiles(\"landvote_state.geojson\", \"landvote_county.geojson\",\"landvote_municipal.geojson\", \"landvote_polygons.pmtiles\")\n",
+    "hf_upload(\"landvote_polygons.pmtiles\", \"boettiger-lab/landvote\")\n"
    ]
   },
   {
@@ -239,11 +249,19 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "df_temp = df_county.union(df_local)\n",
+    "df_temp = df_county.union(df_city)\n",
     "df = df_temp.union(df_state)\n",
-    "df.execute().to_parquet(\"landvote.parquet\")\n",
-    "hf_upload(\"landvote.parquet\", \"boettiger-lab/landvote\")\n"
+    "df.execute().set_crs(\"EPSG:4326\").to_parquet(\"landvote_polygons.parquet\")\n",
+    "hf_upload(\"landvote_polygons.parquet\", \"boettiger-lab/landvote\")\n"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1f1e18bd-dce5-4cd4-b25a-371674340348",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -262,7 +280,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.11.10"
   }
  },
  "nbformat": 4,

get_party.ipynb

@@ -1,8 +1,16 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "f2d27d42-74aa-44cb-8ab6-5a0f856dcca0",
+   "metadata": {},
+   "source": [
+    "# Merging state, county, and city polygons with political parties"
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": null,
    "id": "d179ded1-6235-47ed-bbfb-6d72468188d5",
    "metadata": {},
    "outputs": [],
@@ -12,226 +20,105 @@
     "import streamlit as st\n",
     "import ibis.expr.datatypes as dt  # Make sure to import the necessary module\n",
     "\n",
-    "\n",
-    "conn = ibis.duckdb.connect(extensions=[\"spatial\"])\n",
-    "\n",
-    "# pres = conn.read_csv(\"sources-president.csv\")\n",
-    "county = conn.read_csv(\"countypres_2000-2020.csv\")\n",
-    "votes = conn.read_parquet(\"vote.parquet\")"
+    "conn = ibis.duckdb.connect(extensions=[\"spatial\"])"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "170ba045-8848-4a99-a4f6-68bde22428af",
+   "id": "b9bc2d50-481b-4f62-a74b-4a576ff89ecd",
    "metadata": {},
    "source": [
-    "# Getting party affiliations for counties"
+    "# State "
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "ab644102-c725-4cf4-915c-8550a0a74c32",
+   "id": "322b9a85-bdf9-45f9-9b19-695cc1b996e8",
    "metadata": {},
    "outputs": [],
    "source": [
-    "filtered = county.filter((_.mode == \"TOTAL\") & (_.totalvotes > 0))\n",
-    "\n",
-    "# Find the winning party for each year, state, and county\n",
-    "most_votes = (\n",
-    "    filtered\n",
-    "    .group_by(['year', 'state_po', 'county_name', 'party'])\n",
-    "    .aggregate(winning_votes=_.candidatevotes.sum())\n",
-    ")\n",
-    "\n",
-    "# For each year, state, and county, select the party with the highest total votes\n",
-    "winning_party = (\n",
-    "    most_votes\n",
-    "    .group_by('year', 'state_po', 'county_name')\n",
-    "    .aggregate(\n",
-    "        max_votes=_.winning_votes.max(),  # Max votes in this group\n",
-    "    )\n",
-    "    .join(\n",
-    "        most_votes,\n",
-    "        [\"year\",\"state_po\",\"county_name\",most_votes['winning_votes'] == _.max_votes],\n",
-    "        how='inner'\n",
-    "    )\n",
-    "    .select(\"year\",\"state_po\",\"county_name\",most_votes['party'].name('current_party')\n",
-    "    )\n",
-    ")\n",
-    "\n",
-    "# Self-join to get the previous year's winning party\n",
-    "previous_year = winning_party.view()\n",
-    "\n",
-    "joined = (\n",
-    "    winning_party\n",
-    "    .join(\n",
-    "        previous_year, [\"county_name\",\"state_po\",winning_party['year'] == previous_year['year'] + 4],\n",
-    "        how='left'\n",
-    "    )\n",
-    "    .rename(state_id = \"state_po\")\n",
-    "    .mutate(key = _.county_name + ibis.literal(\" COUNTY-\") + _.state_id)\n",
-    "    .select(\"year\",\"key\",\"current_party\",previous_year['current_party'].name('previous_party'))\n",
-    ")\n",
-    "\n",
-    "county_parties = joined.filter(_.year >2000).order_by(\"year\")\n",
-    "\n",
-    "print(county_parties.execute())"
+    "#getting party\n",
+    "state = (conn\n",
+    "        .read_csv(\"1976-2020-president.csv\")\n",
+    "        .filter(_. year >=2000)\n",
+    "        .rename(state=\"state_po\" , party = \"party_simplified\")  # rename columns\n",
+    "        .group_by([\"year\", \"state\"])\n",
+    "        .aggregate(party=_.party.argmax(_.candidatevotes))  # winning party \n",
+    "        .select(\"year\", \"state\", \"party\")  # select only relevant columns\n",
+    "        )"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "ce0d80bf-3b78-4aa9-8048-5cc0dbf970d9",
+   "id": "6ab49b51-d6fe-47b3-9bed-b23b5ecf7f0e",
    "metadata": {},
    "outputs": [],
    "source": [
-    "df = (votes\n",
-    "        .mutate(key = _.key.upper())\n",
-    "        .filter(_.jurisdiction == \"Municipal\")\n",
-    "        .join(county_parties, [\"key\",\"year\"],how='inner'\n",
-    "        )\n",
-    "        .cast({\"geometry\": \"geometry\"})\n",
-    ")"
+    "# merging with state polygons\n",
+    "state_boundaries = \"https://data.source.coop/cboettig/us-boundaries/us-state-territory.parquet\"\n",
+    "\n",
+    "df_state = (conn\n",
+    "            .read_parquet(state_boundaries)\n",
+    "            .rename(state = \"STUSPS\", state_ = \"NAME\")\n",
+    "            .select(\"state\",\"geometry\")\n",
+    "            .join(state,\"state\",how = \"inner\")\n",
+    "            .mutate(county = None)\n",
+    "            .mutate(municipal = None)\n",
+    "            .mutate(jurisdiction = ibis.literal(\"State\"))\n",
+    "            .cast({\"geometry\": \"geometry\",\"county\":\"string\",\"municipal\": \"string\"})\n",
+    "            .select(\"state\", \"county\", \"municipal\",\"jurisdiction\",\"geometry\", \"year\", \"party\")\n",
+    "            )"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": 30,
-   "id": "87bef5e2-a40a-4aff-aa27-e7d49ec68aac",
+   "cell_type": "markdown",
+   "id": "170ba045-8848-4a99-a4f6-68bde22428af",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n",
-      "Token is valid (permission: write).\n",
-      "Your token has been saved to /home/rstudio/.cache/huggingface/token\n",
-      "Login successful\n"
-     ]
-    }
-   ],
    "source": [
-    "import subprocess\n",
-    "import os\n",
-    "from huggingface_hub import HfApi, login\n",
-    "import streamlit as st\n",
-    "\n",
-    "login(st.secrets[\"HF_TOKEN\"])\n",
-    "# api = HfApi(add_to_git_credential=False)\n",
-    "api = HfApi()\n",
-    "\n",
-    "def hf_upload(file, repo_id):\n",
-    "    info = api.upload_file(\n",
-    "            path_or_fileobj=file,\n",
-    "            path_in_repo=file,\n",
-    "            repo_id=repo_id,\n",
-    "            repo_type=\"dataset\",\n",
-    "        )\n",
-    "def generate_pmtiles(input_file, output_file, max_zoom=12):\n",
-    "    # Ensure Tippecanoe is installed\n",
-    "    if subprocess.call([\"which\", \"tippecanoe\"], stdout=subprocess.DEVNULL) != 0:\n",
-    "        raise RuntimeError(\"Tippecanoe is not installed or not in PATH\")\n",
-    "\n",
-    "    # Construct the Tippecanoe command\n",
-    "    command = [\n",
-    "        \"tippecanoe\",\n",
-    "        \"-o\", output_file,\n",
-    "        \"-zg\",\n",
-    "        \"--extend-zooms-if-still-dropping\",\n",
-    "        \"--force\",\n",
-    "        \"--projection\", \"EPSG:4326\",  \n",
-    "        input_file\n",
-    "    ]\n",
-    "\n",
-    "    # Run Tippecanoe\n",
-    "    try:\n",
-    "        subprocess.run(command, check=True)\n",
-    "        print(f\"Successfully generated PMTiles file: {output_file}\")\n",
-    "    except subprocess.CalledProcessError as e:\n",
-    "        print(f\"Error running Tippecanoe: {e}\")\n",
-    "\n"
+    "# County"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "b086e76c-4285-4036-8033-e4e45cb6966b",
+   "id": "0231c801-82e2-45be-9ec5-607d5588a3e5",
    "metadata": {},
    "outputs": [],
    "source": [
-    "gdf= df.execute()\n",
-    "gdf = gdf.set_crs(\"EPSG:4326\")\n",
-    "\n",
-    "# gdf.to_parquet(\"county_parties.parquet\")\n",
-    "# hf_upload(\"county_parties.parquet\", \"boettiger-lab/landvote\")\n",
-    "\n",
-    "# gdf.to_file(\"county_parties.geojson\")\n",
-    "# hf_upload(\"county_parties.geojson\", \"boettiger-lab/landvote\")\n",
-    "\n",
-    "# generate_pmtiles(\"county_parties.geojson\", \"county_parties.pmtiles\")\n",
-    "# hf_upload(\"county_parties.pmtiles\", \"boettiger-lab/landvote\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "6193c4b9-0183-4aae-9a25-899a748fd65e",
-   "metadata": {},
-   "source": [
-    "# Checking map"
+    "# getting party\n",
+    "county = (conn\n",
+    "        .read_csv(\"countypres_2000-2020.csv\")\n",
+    "        .filter((_.mode == \"TOTAL\") & (_.totalvotes > 0))  # filter only final votes\n",
+    "        .rename(state=\"state_po\", state_name = \"state\") \n",
+    "        .mutate(county = _.county_name + ibis.literal(\" COUNTY\"))\n",
+    "        .group_by([\"year\", \"state\", \"county\",\"state_name\"])\n",
+    "        .aggregate(party=_.party.argmax(_.candidatevotes))  # winning party \n",
+    "        .select(\"year\", \"state\", \"county\", \"party\",\"state_name\")  \n",
+    ")"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "c2ae8ada-c73e-4b2e-938e-70a29584f199",
+   "id": "179b5066-030c-4302-a8cf-8216a753080e",
    "metadata": {},
    "outputs": [],
    "source": [
-    "import leafmap.maplibregl as leafmap\n",
-    "m = leafmap.Map(style=\"positron\")\n",
-    "\n",
-    "\n",
-    "url_states = \"https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/county_parties.pmtiles\"\n",
-    "\n",
-    "outcome = [\n",
-    "      'match',\n",
-    "      ['get', 'Status'], \n",
-    "      \"Pass\", '#2E865F',\n",
-    "      \"Fail\", '#FF3300', \n",
-    "      '#ccc'\n",
-    "    ]\n",
-    "paint_states = {\"fill-color\": outcome, \n",
-    "         # \"fill-opacity\": 0.2,\n",
-    "        }\n",
-    "style_states = {\n",
-    "    \"layers\": [\n",
-    "        {\n",
-    "            \"id\": \"county_parties\",\n",
-    "            \"source\": \"county_parties\",\n",
-    "            \"source-layer\": \"county_parties\",\n",
-    "            \"type\": \"fill\",\n",
-    "            \"filter\": [\n",
-    "                \"==\",\n",
-    "                [\"get\", \"year\"],\n",
-    "                2008,\n",
-    "            ],  # only show buildings with height info\n",
-    "            \"paint\": paint_states\n",
-    "        },\n",
-    "    ],\n",
-    "}\n",
-    "\n",
-    "m.add_pmtiles(\n",
-    "    url_states,\n",
-    "    style=style_states,\n",
-    "    visible=True,\n",
-    "    opacity=0.4,\n",
-    "    tooltip=True,\n",
-    "    fit_bounds=False,\n",
-    ")\n",
-    "\n",
-    "m\n"
+    "# merging with county polygons\n",
+    "county_boundaries = \"https://data.source.coop/cboettig/us-boundaries/us-county.parquet\"\n",
+    "df_county = (conn\n",
+    "        .read_parquet(county_boundaries)\n",
+    "        .mutate(county = _.NAMELSAD.upper(), state_name = _.STATE_NAME.upper())\n",
+    "        .select(\"state_name\",\"county\",\"geometry\")\n",
+    "        .join(county,[\"state_name\",\"county\"],how = \"inner\")\n",
+    "        .mutate(municipal = None)\n",
+    "        .cast({\"geometry\": \"geometry\",\"municipal\": \"string\"})\n",
+    "        .mutate(jurisdiction = ibis.literal(\"County\"))\n",
+    "        .select(\"state\", \"county\", \"municipal\",\"jurisdiction\",\"geometry\", \"year\", \"party\")\n",
+    "\n",
+    "        )"
    ]
   },
   {
@@ -249,227 +136,124 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "\n",
     "localities_boundaries = \"us_localities.parquet\"\n",
-    "locality = conn.read_parquet(localities_boundaries)\n",
-    "landvote = conn.read_csv(\"landvote.csv\")\n",
-    "\n",
-    "#needed to redo this, since I didn't save county in \"votes.parquet\". \n",
-    "vote_local = (landvote\n",
-    "                .filter(_[\"Jurisdiction Type\"] == \"Municipal\")\n",
-    "                .rename(city = \"Jurisdiction Name\", state_id = \"State\")\n",
-    "                .mutate(key = _.city + ibis.literal('-') + _.state_id)\n",
-    "                .rename(amount = 'Conservation Funds at Stake', yes = '% Yes')\n",
-    "                .mutate(amount_n=_.amount.replace('$', '').replace(',', '').cast('float'))\n",
-    "                .mutate(log_amount=_.amount_n.log())\n",
-    "                .mutate(year=_['Date'].year().cast('int32'))\n",
-    "                .mutate(\n",
-    "                    yes=ibis.case()\n",
-    "                        .when(_.yes.isin(['Pass', 'None','Fail']), None)  # Handle non-numeric cases\n",
-    "                        .when(_.yes.notnull(), (_.yes.replace('%', '').cast('float').round(2).cast(dt.float64)).cast(dt.string) + '%')  # Convert valid percentages and add %\n",
-    "                        .else_(None)  # Default to None for other cases\n",
-    "                        .end()\n",
-    "                )\n",
-    "              .mutate(log_amount = _.log_amount.round(4))\n",
-    "                .select('key', 'Status', 'yes', 'year', 'amount', 'log_amount', )\n",
-    "                )\n",
-    "\n",
-    "# getting the county parties for each municipal  \n",
-    "df_municipals = (locality \n",
-    "            .mutate(key_municipal = _.name + ibis.literal('-') + _.state_id) \n",
-    "            .mutate(key = (_.county + ibis.literal('-') + _.state_id).upper()) \n",
-    "            .select('key', 'geometry','key_municipal','name')\n",
-    "            .right_join(vote_local, [_.key_municipal == vote_local[\"key\"]])\n",
-    "            .mutate(jurisdiction = ibis.literal(\"Municipal\"))\n",
-    "            .cast({\"geometry\": \"geometry\"})\n",
-    "            .mutate(geometry = _.geometry.buffer(.07))\n",
-    "            .join(county_parties, [\"key\",\"year\"],how='inner')\n",
-    "            .rename(county = \"key\")\n",
-    "            .rename(key = \"key_municipal\")\n",
-    "            .select('key','geometry','Status','yes','year','amount','log_amount','jurisdiction','current_party','previous_party')\n",
+    "locality = (conn\n",
+    "            .read_parquet(localities_boundaries)\n",
+    "            .mutate(county = _.county.upper())\n",
+    "            .mutate(municipal = _.municipal.upper())\n",
     "           )\n",
     "\n",
-    "\n",
-    "gdf_municipals = df_municipals.execute()\n",
-    "gdf_municipals = gdf_municipals.set_crs(\"EPSG:4326\")\n",
-    "gdf_municipals\n",
-    "\n",
-    "\n",
-    "# gdf_municipals.to_parquet(\"municipal_parties.parquet\")\n",
-    "# hf_upload(\"municipal_parties.parquet\", \"boettiger-lab/landvote\")\n",
-    "\n",
-    "# gdf_municipals.to_file(\"municipal_parties.geojson\")\n",
-    "# hf_upload(\"municipal_parties.geojson\", \"boettiger-lab/landvote\")\n",
-    "\n",
-    "# generate_pmtiles(\"municipal_parties.geojson\", \"municipal_parties.pmtiles\")\n",
-    "# hf_upload(\"municipal_parties.pmtiles\", \"boettiger-lab/landvote\")\n"
+    "df_city = (county\n",
+    "            .drop(\"state_name\")\n",
+    "            .join(locality, [\"state\",\"county\"], how = \"inner\")\n",
+    "            .cast({\"geometry\": \"geometry\"})\n",
+    "            .mutate(jurisdiction = ibis.literal(\"Municipal\"))\n",
+    "            .select(\"state\", \"county\", \"municipal\",\"jurisdiction\",\"geometry\", \"year\", \"party\")\n",
+    "            )\n"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "06e24a7e-5f7f-42bc-b515-43082016d496",
+   "id": "ae5b417d-4266-456d-952c-ac2696234ea0",
    "metadata": {},
    "source": [
-    "# Get States"
+    "# Make PMTiles. Each jurisdiction type is its own layer"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
-   "id": "217170ca-b732-4875-b4cb-f8a8cd2fc405",
+   "execution_count": null,
+   "id": "12cdf02e-bc22-4a5f-91b9-00a8eee587bd",
    "metadata": {},
    "outputs": [],
    "source": [
-    "states = (conn\n",
-    "          .read_csv(\"1976-2020-president.csv\")\n",
-    "          .filter(_. year >=2000)\n",
-    "         )\n",
-    "# states.execute()"
+    "import subprocess\n",
+    "import os\n",
+    "from huggingface_hub import HfApi, login\n",
+    "import streamlit as st\n",
+    "\n",
+    "login(st.secrets[\"HF_TOKEN\"])\n",
+    "# api = HfApi(add_to_git_credential=False)\n",
+    "api = HfApi()\n",
+    "\n",
+    "def hf_upload(file, repo_id):\n",
+    "    info = api.upload_file(\n",
+    "            path_or_fileobj=file,\n",
+    "            path_in_repo=file,\n",
+    "            repo_id=repo_id,\n",
+    "            repo_type=\"dataset\",\n",
+    "        )\n",
+    "def generate_pmtiles(input_file, input_file2, input_file3, output_file, max_zoom=12):\n",
+    "    # Ensure Tippecanoe is installed\n",
+    "    if subprocess.call([\"which\", \"tippecanoe\"], stdout=subprocess.DEVNULL) != 0:\n",
+    "        raise RuntimeError(\"Tippecanoe is not installed or not in PATH\")\n",
+    "\n",
+    "    # Construct the Tippecanoe command\n",
+    "    command = [\n",
+    "        \"tippecanoe\",\n",
+    "        \"-o\", output_file,\n",
+    "        \"-zg\",\n",
+    "        \"--extend-zooms-if-still-dropping\",\n",
+    "        \"--force\",\n",
+    "        \"--projection\", \"EPSG:4326\",  \n",
+    "        \"-L\",\"state:\"+input_file,\n",
+    "        \"-L\",\"county:\"+input_file2,\n",
+    "        \"-L\",\"municipal:\"+input_file3\n",
+    "    ]\n",
+    "    # Run Tippecanoe\n",
+    "    try:\n",
+    "        subprocess.run(command, check=True)\n",
+    "        print(f\"Successfully generated PMTiles file: {output_file}\")\n",
+    "    except subprocess.CalledProcessError as e:\n",
+    "        print(f\"Error running Tippecanoe: {e}\")\n",
+    "\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
-   "id": "322b9a85-bdf9-45f9-9b19-695cc1b996e8",
+   "execution_count": null,
+   "id": "24df77fb-c881-4491-b7ca-7f3a3023cee0",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "     year key current_party previous_party\n",
-      "0    2004  HI      DEMOCRAT       DEMOCRAT\n",
-      "1    2004  ME      DEMOCRAT       DEMOCRAT\n",
-      "2    2004  NJ      DEMOCRAT       DEMOCRAT\n",
-      "3    2004  NM    REPUBLICAN       DEMOCRAT\n",
-      "4    2004  ND    REPUBLICAN     REPUBLICAN\n",
-      "..    ...  ..           ...            ...\n",
-      "250  2020  VT      DEMOCRAT       DEMOCRAT\n",
-      "251  2020  AL    REPUBLICAN     REPUBLICAN\n",
-      "252  2020  IA    REPUBLICAN     REPUBLICAN\n",
-      "253  2020  SD    REPUBLICAN     REPUBLICAN\n",
-      "254  2020  GA      DEMOCRAT     REPUBLICAN\n",
-      "\n",
-      "[255 rows x 4 columns]\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "# filtered = county.filter((_.mode == \"TOTAL\") & (_.totalvotes > 0))\n",
+    "gdf_state = df_state.execute().set_crs(\"EPSG:4326\")\n",
+    "gdf_state.to_file(\"party_state.geojson\")\n",
     "\n",
-    "# Find the winning party for each year, state, and county\n",
-    "most_votes= (\n",
-    "    states\n",
-    "    .group_by(['year', 'state_po', 'party_simplified'])\n",
-    "    .aggregate(winning_votes=_.candidatevotes.sum())\n",
-    ")\n",
+    "gdf_county = df_county.execute().set_crs(\"EPSG:4326\")\n",
+    "gdf_county.to_file(\"party_county.geojson\")\n",
     "\n",
-    "# For each year, state, and county, select the party with the highest total votes\n",
-    "winning_party = (\n",
-    "    most_votes\n",
-    "    .group_by('year', 'state_po')\n",
-    "    .aggregate(\n",
-    "        max_votes=_.winning_votes.max(),  # Max votes in this group\n",
-    "    )\n",
-    "    .join(\n",
-    "        most_votes,\n",
-    "        [\"year\",\"state_po\",most_votes['winning_votes'] == _.max_votes],\n",
-    "        how='inner'\n",
-    "    )\n",
-    "    .select(\"year\",\"state_po\",most_votes['party_simplified'].name('current_party')\n",
-    "    )\n",
-    ")\n",
+    "gdf_city = df_city.execute().set_crs(\"EPSG:4326\")\n",
+    "gdf_city.to_file(\"party_municipal.geojson\")\n",
     "\n",
-    "# Self-join to get the previous year's winning party\n",
-    "previous_year = winning_party.view()\n",
-    "\n",
-    "joined = (\n",
-    "    winning_party\n",
-    "    .join(\n",
-    "        previous_year, [\"state_po\",winning_party['year'] == previous_year['year'] + 4],\n",
-    "        how='left'\n",
-    "    )\n",
-    "    .rename(key = \"state_po\")\n",
-    "    # .mutate(key = _.county_name + ibis.literal(\" COUNTY-\") + _.state_id)\n",
-    "    .select(\"year\",\"key\",\"current_party\",previous_year['current_party'].name('previous_party'))\n",
-    ")\n",
-    "\n",
-    "state_parties = joined.filter(_.year >2000).order_by(\"year\")\n",
-    "\n",
-    "print(state_parties.execute())"
+    "generate_pmtiles(\"party_state.geojson\", \"party_county.geojson\",\"party_municipal.geojson\", \"landvote_polygons.pmtiles\")\n",
+    "hf_upload(\"party_polygons.pmtiles\", \"boettiger-lab/landvote\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "190169bb-5bfb-4eb7-a135-c5ce0e316595",
+   "metadata": {},
+   "source": [
+    "# Combine all 3 jurisdiction types into a parquet."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
-   "id": "2c03920e-76da-4034-8eaf-1e80a56f5b0d",
+   "execution_count": null,
+   "id": "4ad8ad0a-afb3-427f-8b52-ea328e06ce85",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "3574af6546ff4cd1949e27c63cd15cd7",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "states_parties.parquet:   0%|          | 0.00/2.36M [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "For layer 0, using name \"states_parties\"\n",
-      "12 features, 833591 bytes of geometry and attributes, 542 bytes of string pool, 0 bytes of vertices, 0 bytes of nodes\n",
-      "Choosing a maxzoom of -z0 for features typically 7514540 feet (2290432 meters) apart, and at least 2073685 feet (632060 meters) apart\n",
-      "Choosing a maxzoom of -z10 for resolution of about 376 feet (114 meters) within features\n",
-      "  99.9%  10/271/383  \n",
-      "  100.0%  10/187/380  \r"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Successfully generated PMTiles file: states_parties.pmtiles\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "# state_boundaries = \"https://data.source.coop/cboettig/us-boundaries/us-state-territory.parquet\"\n",
-    "states = conn.read_parquet(\"vote_states.parquet\")\n",
-    "\n",
-    "df_states = (states\n",
-    "        .mutate(key = _.key.upper())\n",
-    "        # .filter(_.jurisdiction == \"Municipal\")\n",
-    "        .join(state_parties, [\"key\",\"year\"],how='inner'\n",
-    "        )\n",
-    "        .cast({\"geometry\": \"geometry\"})\n",
-    ")\n",
-    "\n",
-    "gdf_states = df_states.execute()\n",
-    "gdf_states = gdf_states.set_crs(\"EPSG:4326\")\n",
-    "gdf_states\n",
-    "\n",
-    "\n",
-    "gdf_states.to_parquet(\"states_parties.parquet\")\n",
-    "hf_upload(\"states_parties.parquet\", \"boettiger-lab/landvote\")\n",
-    "\n",
-    "gdf_states.to_file(\"states_parties.geojson\")\n",
-    "hf_upload(\"states_parties.geojson\", \"boettiger-lab/landvote\")\n",
-    "\n",
-    "generate_pmtiles(\"states_parties.geojson\", \"states_parties.pmtiles\")\n",
-    "hf_upload(\"states_parties.pmtiles\", \"boettiger-lab/landvote\")\n"
+    "df_temp = df_county.union(df_city)\n",
+    "df = df_temp.union(df_state)\n",
+    "df.execute().set_crs(\"EPSG:4326\").to_parquet(\"party_polygons.parquet\")\n",
+    "hf_upload(\"party_polygons.parquet\", \"boettiger-lab/landvote\")\n"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "dcf5d049-68ae-4d73-be77-72d985c9ed1c",
+   "id": "3e87fc40-b535-4a44-9230-99f6777e7c48",
    "metadata": {},
    "outputs": [],
    "source": []
@@ -491,7 +275,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.11.10"
   }
  },
  "nbformat": 4,

preprocess.ipynb

@@ -1,84 +1,118 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "9fe02137-ba1c-4ced-9909-b5b7c39ed6d4",
+   "metadata": {},
+   "source": [
+    "# Merging state/county/city polygons with party affiliation and landvote data"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "3e5756d2-382b-49e9-93b5-2ecf6d0eb812",
+   "id": "6452373c-d10f-472c-9174-cd05a3363587",
    "metadata": {},
    "outputs": [],
    "source": [
-    "import duckdb\n",
+    "import ibis\n",
+    "from ibis import _\n",
     "\n",
-    "con = duckdb.connect()\n",
+    "import streamlit as st\n",
+    "import ibis.expr.datatypes as dt  # Make sure to import the necessary module\n",
     "\n",
-    "con.execute(\"SET s3_region='us-west-2';\")\n",
-    "con.execute(\"LOAD spatial;\")\n",
-    "con.execute(\"LOAD httpfs;\")\n",
+    "conn = ibis.duckdb.connect(extensions=[\"spatial\"])\n",
     "\n",
-    "query = \"\"\"\n",
-    "    COPY (\n",
-    "        SELECT * \n",
-    "        FROM read_parquet('s3://overturemaps-us-west-2/release/2024-09-18.0/theme=divisions/*/*')\n",
-    "        WHERE country = 'US' AND subtype IN ('locality', 'neighborhood')\n",
-    "    ) TO 'us_localities_neighborhoods.parquet' (FORMAT 'parquet');\n",
-    "\"\"\"\n",
-    "con.execute(query)\n",
+    "landvote_url = \"https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/landvote_polygons.parquet\"\n",
+    "party_url = \"https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/party_polygons.parquet\"\n",
     "\n"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "25f62dd7-5539-438b-8f0a-1d85c9bc78ab",
+   "id": "371ef7f4-95b9-49f0-80ba-48807b98b052",
    "metadata": {},
    "outputs": [],
    "source": [
-    "import ibis\n",
-    "from ibis import _\n",
-    "\n",
-    "conn = ibis.duckdb.connect(extensions=[\"spatial\"])\n",
-    "\n",
-    "df = (conn\n",
-    "      .read_parquet(\"us_localities_neighborhoods.parquet\")\n",
-    "      .cast({\"geometry\": \"geometry\"})\n",
-    "      .filter(_[\"type\"] == \"division\")\n",
-    "      .filter(_[\"subtype\"] == \"locality\")\n",
-    "      .mutate(name = _.names[\"primary\"])\n",
-    "      .mutate(state_id = _.region.replace(\"US-\", \"\")) \n",
-    "      .mutate(county = _.hierarchies[0][2]['name'] )\n",
-    "      .mutate(key_long = _.name + ibis.literal('-') + _.county + ibis.literal('-') + _.state_id)\n",
-    "      .select(\"key_long\",\"name\", \"county\",\"state_id\" ,\"geometry\")\n",
-    "     )\n",
-    "\n",
-    "\n",
-    "## Dropping rows with same locality and state, with differing counties \n",
-    "county_count = (\n",
-    "    df.group_by([\"name\", \"state_id\"])\n",
-    "    .aggregate(county_count=_.county.nunique())  # Count unique counties for each group\n",
-    ") \n",
-    "valid_names = county_count.filter(county_count.county_count == 1).select(\"name\", \"state_id\")\n",
-    "df_filtered = df.join(valid_names, [\"name\", \"state_id\"], how=\"inner\")\n",
+    "landvote = (conn\n",
+    "            .read_parquet(landvote_url)\n",
+    "            .cast({\"geometry\": \"geometry\"})\n",
+    "            .mutate(county = _.county.upper())\n",
+    "            .mutate(municipal = _.municipal.upper())\n",
+    "            .mutate(elect_year = _.year - _.year % 4) # get most recent election year \n",
+    "            .cast({\"municipal\": \"string\",\"county\":\"string\"})\n",
+    "            .mutate(municipal=ibis.case()\n",
+    "                    .when(_.jurisdiction.isin(['State','County']), ibis.literal(\"-\")) \n",
+    "                    .else_(_.municipal) \n",
+    "                    .end()\n",
+    "                    )\n",
+    "                    .mutate(county=ibis.case()\n",
+    "                    .when(_.jurisdiction.isin(['State']),ibis.literal(\"-\")) \n",
+    "                    .else_(_.county) \n",
+    "                    .end()\n",
+    "                    )\n",
+    "            )\n",
     "\n",
-    "\n",
-    "# if two records have the same name but different geometries, only keep the first one.\n",
-    "df_first = (\n",
-    "    df_filtered.group_by(\"key_long\")\n",
-    "    .aggregate(\n",
-    "        name=df_filtered.name.first(),\n",
-    "        county=df_filtered.county.first(),\n",
-    "        state_id=df_filtered.state_id.first(),\n",
-    "        geometry=df_filtered.geometry.first()\n",
-    "    )\n",
-    "\n",
-    ")\n",
-    "\n",
-    "df_first.execute().to_parquet(\"us_localities.parquet\")\n"
+    "party = (conn\n",
+    "        .read_parquet(party_url)\n",
+    "        .cast({\"geometry\": \"geometry\"})\n",
+    "        .mutate(municipal=ibis.case()\n",
+    "                .when(_.jurisdiction.isin(['State','County']), ibis.literal(\"-\")) \n",
+    "                .else_(_.municipal) \n",
+    "                .end()\n",
+    "                )\n",
+    "        .mutate(county=ibis.case()\n",
+    "                .when(_.jurisdiction.isin(['State']), ibis.literal(\"-\")) \n",
+    "                .else_(_.county) \n",
+    "                .end()\n",
+    "                )\n",
+    "        )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8fc40e96-fffd-4b23-9963-c931fdce96f6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "votes = (landvote\n",
+    "        .join(party,[\"state\",\"county\",\"municipal\",\"jurisdiction\",'geometry', _.elect_year == party[\"year\"]],how = \"inner\")\n",
+    "        .drop('elect_year','year_right')\n",
+    "        .mutate(municipal=ibis.case()\n",
+    "                .when(_.municipal == ibis.literal(\"-\"), None) \n",
+    "                .else_(_.municipal) \n",
+    "                .end()\n",
+    "                )\n",
+    "        .mutate(county=ibis.case()\n",
+    "                .when(_.county == ibis.literal(\"-\"), None) \n",
+    "                .else_(_.county) \n",
+    "                .end()\n",
+    "                )\n",
+    "        )"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "ca02743f-0bf4-46e5-91fd-a5fe37519ecd",
+   "id": "65e75bc0-fef0-48f2-a543-7aae999579bf",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e80cfd2e-40eb-4065-9ae6-dcaf83319d9a",
+   "metadata": {},
+   "source": [
+    "# Make PMTiles. Each jurisdiction type is its own layer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b1cd8f44-57fa-49a8-b438-f9e4aab747c5",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -98,9 +132,71 @@
     "            repo_id=repo_id,\n",
     "            repo_type=\"dataset\",\n",
     "        )\n",
-    "hf_upload(\"us_localities.parquet\", \"boettiger-lab/landvote\")\n",
+    "def generate_pmtiles(input_file, input_file2, input_file3, output_file, max_zoom=12):\n",
+    "    # Ensure Tippecanoe is installed\n",
+    "    if subprocess.call([\"which\", \"tippecanoe\"], stdout=subprocess.DEVNULL) != 0:\n",
+    "        raise RuntimeError(\"Tippecanoe is not installed or not in PATH\")\n",
+    "\n",
+    "    # Construct the Tippecanoe command\n",
+    "    command = [\n",
+    "        \"tippecanoe\",\n",
+    "        \"-o\", output_file,\n",
+    "        \"-zg\",\n",
+    "        \"--extend-zooms-if-still-dropping\",\n",
+    "        \"--force\",\n",
+    "        \"--projection\", \"EPSG:4326\",  \n",
+    "        \"-L\",\"state:\"+input_file,\n",
+    "        \"-L\",\"county:\"+input_file2,\n",
+    "        \"-L\",\"municipal:\"+input_file3\n",
+    "    ]\n",
+    "    # Run Tippecanoe\n",
+    "    try:\n",
+    "        subprocess.run(command, check=True)\n",
+    "        print(f\"Successfully generated PMTiles file: {output_file}\")\n",
+    "    except subprocess.CalledProcessError as e:\n",
+    "        print(f\"Error running Tippecanoe: {e}\")\n",
     "\n"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7061577e-0632-4395-8ac5-241a1fab53b0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gdf_state = votes.filter(_.jurisdiction == 'State').execute().set_crs(\"EPSG:4326\")\n",
+    "gdf_state.to_file(\"votes_state.geojson\")\n",
+    "\n",
+    "gdf_county = votes.filter(_.jurisdiction == 'County').execute().set_crs(\"EPSG:4326\")\n",
+    "gdf_county.to_file(\"votes_county.geojson\")\n",
+    "\n",
+    "gdf_city = votes.filter(_.jurisdiction == 'Municipal').execute().set_crs(\"EPSG:4326\")\n",
+    "gdf_city.to_file(\"votes_municipal.geojson\")\n",
+    "\n",
+    "generate_pmtiles(\"votes_state.geojson\", \"votes_county.geojson\",\"votes_municipal.geojson\", \"votes.pmtiles\")\n",
+    "hf_upload(\"votes.pmtiles\", \"boettiger-lab/landvote\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f2979624-bcdf-4a8a-899a-c22fc3cdaf0e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save as parquet\n",
+    "votes.execute().set_crs(\"EPSG:4326\").to_parquet(\"votes.parquet\")\n",
+    "hf_upload(\"votes.parquet\", \"boettiger-lab/landvote\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2ec22cf4-cfdc-4845-a793-ed9236054ff4",
+   "metadata": {},
+   "source": [
+    "# "
+   ]
   }
  ],
  "metadata": {