adding layers to pmtiles

app.py

@@ -63,10 +63,7 @@ year = st.slider("Select a year", min_value=1988, max_value=2024, value=2022, st
 import leafmap.maplibregl as leafmap
 m = leafmap.Map(style="positron",   center=(-100, 40), zoom=3)
 
-url = "https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/vote.pmtiles"
-
-url_states = "https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/vote_states.pmtiles"
-
+url = "https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/landvote.pmtiles"
 
 dark_orange = 'rgba(171, 86, 1, 1)'   # dark orange - min value 
 light_orange = 'rgba(243, 211, 177, 1)' # light orange 
@@ -75,12 +72,55 @@ light_green = 'rgba(195, 219, 195, 1)' # light green
 dark_green = 'rgba(65, 125, 65, 1)'    # dark green  - max value 
 
 
-style = {
+style_municipals = {
+    "layers": [
+        {
+            "id": "municipals",
+            "source": "municipal",
+            "source-layer": "municipal",
+            "type": "fill-extrusion",
+            "filter": [
+                "==",
+                ["get", "year"],
+                year,
+            ],
+            "paint": {
+                "fill-extrusion-color": [
+                    "case",
+                    # if passed, color green
+                    ["==", ["get", "Status"], "Pass"],
+                    [
+                        "interpolate", ["linear"], [
+                            "to-number", ["slice", ["get", "yes"], 0, -1]  # convert 'yes' string to number
+                        ],
+                        50, grey,    
+                        55, light_green,     # higher yes % -> darker green
+                        100, dark_green     # 100 is the max of data 
+                    ],
+                    # if failed, color orange
+                    ["==", ["get", "Status"], "Fail"],
+                    [
+                        "interpolate", ["linear"], [
+                            "to-number", ["slice", ["get", "yes"], 0, -1]  # convert 'yes' string to number
+                        ],
+                        0, dark_orange,     # higher yes % -> lighter orange 
+                        50, light_orange,   
+                        67, grey # 67 is the max of data.
+                    ],
+                    grey  # if no match 
+                ],
+                "fill-extrusion-height": ["*", ["get", "log_amount"], 5000],
+            }
+        },
+    ],
+}
+
+style_counties = {
     "layers": [
         {
-            "id": "counties & municipals",
-            "source": "vote",
-            "source-layer": "vote",
+            "id": "counties",
+            "source": "county",
+            "source-layer": "county",
             "type": "fill-extrusion",
             "filter": [
                 "==",
@@ -122,8 +162,8 @@ style_states = {
     "layers": [
         {
             "id": "states",
-            "source": "vote_states",
-            "source-layer": "vote_states",
+            "source": "state",
+            "source-layer": "state",
             "type": "fill",
             "filter": [
                 "==",
@@ -163,7 +203,7 @@ style_states = {
 
 #states are 2D and transparent, thus added separately.
 m.add_pmtiles(
-    url_states,
+    url,
     style=style_states,
     visible=True,
     opacity=0.6,
@@ -171,9 +211,20 @@ m.add_pmtiles(
     fit_bounds=False
 )
 
+#states are 2D and transparent, thus added separately.
+m.add_pmtiles(
+    url,
+    style=style_counties,
+    visible=True,
+    opacity=0.6,
+    tooltip=True,
+    fit_bounds=False
+)
+
+
 m.add_pmtiles(
     url,
-    style=style,
+    style=style_municipals,
     visible=True,
     opacity=1.0,
     tooltip=True,

static-maps.ipynb

@@ -12,31 +12,28 @@
     "import streamlit as st\n",
     "import ibis.expr.datatypes as dt  # Make sure to import the necessary module\n",
     "\n",
-    "\n",
-    "conn = ibis.duckdb.connect(extensions=[\"spatial\"])\n",
-    "\n",
-    "state_boundaries = \"https://data.source.coop/cboettig/us-boundaries/us-state-territory.parquet\"\n",
-    "county_boundaries = \"https://data.source.coop/cboettig/us-boundaries/us-county.parquet\"\n",
-    "states = conn.read_parquet(state_boundaries).rename(state_id = \"STUSPS\", state = \"NAME\")\n",
-    "county = conn.read_parquet(county_boundaries).rename(county = \"NAMELSAD\", state = \"STATE_NAME\")\n",
-    "\n",
-    "localities_boundaries = \"us_localities.parquet\"\n",
-    "locality = conn.read_parquet(localities_boundaries)\n",
-    "\n",
-    "\n",
-    "votes = conn.read_csv(\"landvote.csv\")"
+    "conn = ibis.duckdb.connect(extensions=[\"spatial\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "943e4127-4af5-42c9-b1e2-48af2d888c24",
+   "metadata": {},
+   "source": [
+    "# Landvote data"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "09ecf89a-a315-45ef-9ece-783963e6e07e",
+   "id": "b486df27-e64c-48af-a577-89a74752ed5b",
    "metadata": {},
    "outputs": [],
    "source": [
-    "vote_states = (votes\n",
-    "                .filter(_[\"Jurisdiction Type\"] == \"State\")\n",
-    "                .rename(state_id = \"State\")\n",
+    "landvote = conn.read_csv(\"landvote.csv\")\n",
+    "\n",
+    "landvote_df = (landvote\n",
+    "                .rename(jurisdiction = \"Jurisdiction Type\", state = \"State\", name = \"Jurisdiction Name\")\n",
     "                .rename(amount = 'Conservation Funds at Stake', yes = '% Yes')\n",
     "                .mutate(amount_n=_.amount.replace('$', '').replace(',', '').cast('float'))\n",
     "                .mutate(log_amount=_.amount_n.log())\n",
@@ -48,18 +45,57 @@
     "                        .else_(None)  # Default to None for other cases\n",
     "                        .end()\n",
     "                )\n",
+    "                .mutate(\n",
+    "                    municipal=ibis.case()\n",
+    "                        .when(_.jurisdiction.isin(['State','County']), None) \n",
+    "                        .else_(_.name) \n",
+    "                        .end()\n",
+    "                )\n",
+    "                .mutate(\n",
+    "                    county=ibis.case()\n",
+    "                        .when(_.jurisdiction.isin(['State','Municipal']), None)  \n",
+    "                        .else_(_.name) \n",
+    "                        .end()\n",
+    "                )\n",
     "               .mutate(log_amount = _.log_amount.round(4))\n",
-    "               .select('state_id', 'Status', 'yes', 'year', 'amount', 'log_amount', )\n",
-    "               )\n",
+    "               .select('name','state','county','municipal','jurisdiction','Status', 'yes', 'year', 'amount', 'log_amount', )\n",
+    "               )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b78ad1e0-3b4e-4f56-bd25-b8da186d2857",
+   "metadata": {},
+   "source": [
+    "# State Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "09ecf89a-a315-45ef-9ece-783963e6e07e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "state_boundaries = \"https://data.source.coop/cboettig/us-boundaries/us-state-territory.parquet\"\n",
+    "states = conn.read_parquet(state_boundaries).rename(state_id = \"STUSPS\", state = \"NAME\")\n",
+    "\n",
+    "landvote_states = landvote_df.filter(_.jurisdiction == \"State\")\n",
     "\n",
-    "df_states = (states\n",
+    "df_state = (states\n",
     "            .select('state_id','geometry')\n",
-    "            .right_join(vote_states, \"state_id\")\n",
-    "            .mutate(key = _.state_id)\n",
-    "            .select('key','geometry','Status', 'yes', 'year', 'amount', 'log_amount')\n",
-    "            .mutate(jurisdiction = ibis.literal(\"State\"))\n",
     "            .cast({\"geometry\": \"geometry\"})\n",
-    "             )\n"
+    "            .right_join(landvote_states, landvote_states['state'] == states['state_id'] )\n",
+    "            .select('state','county','municipal','jurisdiction','geometry','Status', 'yes', 'year', 'amount', 'log_amount', )\n",
+    "             )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "010717bb-1d7e-4915-b476-a4038728131c",
+   "metadata": {},
+   "source": [
+    "# County Data"
    ]
   },
   {
@@ -69,33 +105,26 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "vote_county = (votes\n",
-    "                .filter(_[\"Jurisdiction Type\"] == \"County\")\n",
-    "                .rename(county = \"Jurisdiction Name\", state_id = \"State\")\n",
-    "                .mutate(key = _.county + ibis.literal('-') + _.state_id)\n",
-    "                .rename(amount = 'Conservation Funds at Stake', yes = '% Yes')\n",
-    "                .mutate(amount_n=_.amount.replace('$', '').replace(',', '').cast('float'))\n",
-    "                .mutate(log_amount=_.amount_n.log())\n",
-    "                .mutate(year=_['Date'].year().cast('int32'))\n",
-    "                .mutate(\n",
-    "                    yes=ibis.case()\n",
-    "                        .when(_.yes.isin(['Pass', 'None','Fail']), None)  # Handle non-numeric cases\n",
-    "                        .when(_.yes.notnull(), (_.yes.replace('%', '').cast('float').round(2).cast(dt.float64)).cast(dt.string) + '%')  # Convert valid percentages and add %\n",
-    "                        .else_(None)  # Default to None for other cases\n",
-    "                        .end()\n",
-    "                )\n",
-    "               .mutate(log_amount = _.log_amount.round(4))\n",
-    "                .select('key', 'Status', 'yes', 'year', 'amount', 'log_amount', )\n",
-    "               )\n",
+    "landvote_county = landvote_df.filter(_.jurisdiction == \"County\")\n",
+    "\n",
+    "county_boundaries = \"https://data.source.coop/cboettig/us-boundaries/us-county.parquet\"\n",
+    "county = conn.read_parquet(county_boundaries).rename(county = \"NAMELSAD\", state = \"STATE_NAME\")\n",
+    "\n",
+    "\n",
     "df_county = (county\n",
-    "            .join(states.select(\"state\", \"state_id\"), \"state\")\n",
-    "            .mutate(key = _.county + ibis.literal('-') + _.state_id)\n",
-    "            .select('key', 'geometry')\n",
-    "            .right_join(vote_county, \"key\")\n",
-    "            .drop('key_right')\n",
-    "            .mutate(jurisdiction = ibis.literal(\"County\"))\n",
+    "            .select('state','county','geometry')\n",
     "            .cast({\"geometry\": \"geometry\"})\n",
-    "             )\n"
+    "            .right_join(landvote_county, \"county\" )\n",
+    "            .select('state','county','municipal','jurisdiction','geometry','Status', 'yes', 'year', 'amount', 'log_amount', )\n",
+    "             )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "913b4a98-82eb-452f-8d81-8bce2b58a4c3",
+   "metadata": {},
+   "source": [
+    "# Municipal Data"
    ]
   },
   {
@@ -105,44 +134,28 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "vote_local = (votes\n",
-    "                .filter(_[\"Jurisdiction Type\"] == \"Municipal\")\n",
-    "                .rename(city = \"Jurisdiction Name\", state_id = \"State\")\n",
-    "                .mutate(key = _.city + ibis.literal('-') + _.state_id)\n",
-    "                .rename(amount = 'Conservation Funds at Stake', yes = '% Yes')\n",
-    "                .mutate(amount_n=_.amount.replace('$', '').replace(',', '').cast('float'))\n",
-    "                .mutate(log_amount=_.amount_n.log())\n",
-    "                .mutate(year=_['Date'].year().cast('int32'))\n",
-    "                .mutate(\n",
-    "                    yes=ibis.case()\n",
-    "                        .when(_.yes.isin(['Pass', 'None','Fail']), None)  # Handle non-numeric cases\n",
-    "                        .when(_.yes.notnull(), (_.yes.replace('%', '').cast('float').round(2).cast(dt.float64)).cast(dt.string) + '%')  # Convert valid percentages and add %\n",
-    "                        .else_(None)  # Default to None for other cases\n",
-    "                        .end()\n",
-    "                )\n",
-    "              .mutate(log_amount = _.log_amount.round(4))\n",
-    "                .select('key', 'Status', 'yes', 'year', 'amount', 'log_amount', )\n",
-    "                )\n",
+    "landvote_local = landvote_df.filter(_.jurisdiction == \"Municipal\")\n",
     "\n",
-    "df_local = (locality\n",
-    "            .mutate(key = _.name + ibis.literal('-') + _.state_id)\n",
-    "            .select('key', 'geometry')\n",
-    "            .right_join(vote_local, \"key\")\n",
-    "            .drop('key_right')\n",
-    "            .mutate(jurisdiction = ibis.literal(\"Municipal\"))\n",
+    "local = (conn\n",
+    "        .read_parquet(\"us_localities.parquet\")\n",
+    "        .rename(municipal = \"name\",state = \"state_id\")\n",
+    "        )\n",
+    "\n",
+    "df_local = (local\n",
+    "            .select('state','county','municipal','geometry')\n",
     "            .cast({\"geometry\": \"geometry\"})\n",
     "            .mutate(geometry = _.geometry.buffer(.07))\n",
-    "            )\n"
+    "            .right_join(landvote_local, [\"municipal\",\"state\"])\n",
+    "            .select('state','county','municipal','jurisdiction','geometry','Status', 'yes', 'year', 'amount', 'log_amount', )\n",
+    "             )"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "a1e81807-8ce3-44bf-9a1c-8563fa33817c",
+   "cell_type": "markdown",
+   "id": "728993c4-7d58-4e89-924d-72dd43333585",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "df = df_county.union(df_local)"
+    "# Make PMTiles. Each jurisdiction type is its own layer"
    ]
   },
   {
@@ -168,7 +181,7 @@
     "            repo_id=repo_id,\n",
     "            repo_type=\"dataset\",\n",
     "        )\n",
-    "def generate_pmtiles(input_file, output_file, max_zoom=12):\n",
+    "def generate_pmtiles(input_file, input_file2, input_file3, output_file, max_zoom=12):\n",
     "    # Ensure Tippecanoe is installed\n",
     "    if subprocess.call([\"which\", \"tippecanoe\"], stdout=subprocess.DEVNULL) != 0:\n",
     "        raise RuntimeError(\"Tippecanoe is not installed or not in PATH\")\n",
@@ -181,9 +194,10 @@
     "        \"--extend-zooms-if-still-dropping\",\n",
     "        \"--force\",\n",
     "        \"--projection\", \"EPSG:4326\",  \n",
-    "        input_file\n",
+    "        \"-L\",\"state:\"+input_file,\n",
+    "        \"-L\",\"county:\"+input_file2,\n",
+    "        \"-L\",\"municipal:\"+input_file3\n",
     "    ]\n",
-    "\n",
     "    # Run Tippecanoe\n",
     "    try:\n",
     "        subprocess.run(command, check=True)\n",
@@ -196,94 +210,40 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "902ece1d-6e57-4f83-9286-709dbb549fae",
+   "id": "26ffccfd-2666-445a-9c03-2089bd589650",
    "metadata": {},
    "outputs": [],
    "source": [
-    "gdf= df.execute()\n",
-    "gdf = gdf.set_crs(\"EPSG:4326\")\n",
-    "\n",
-    "gdf.to_file(\"vote.geojson\")\n",
-    "hf_upload(\"vote.geojson\", \"boettiger-lab/landvote\")\n",
+    "df_state.execute().to_file(\"vote_state.geojson\")\n",
+    "df_county.execute().to_file(\"vote_county.geojson\")\n",
+    "df_local.execute().to_file(\"vote_municipal.geojson\")\n",
     "\n",
-    "generate_pmtiles(\"vote.geojson\", \"vote.pmtiles\")\n",
-    "hf_upload(\"vote.pmtiles\", \"boettiger-lab/landvote\")\n",
+    "# hf_upload(\"vote.geojson\", \"boettiger-lab/landvote\")\n",
     "\n",
-    "\n",
-    "\n",
-    "gdf_states= df_states.execute()\n",
-    "gdf_states = gdf_states.set_crs(\"EPSG:4326\")\n",
-    "\n",
-    "gdf_states.to_file(\"vote_states.geojson\")\n",
-    "hf_upload(\"vote_states.geojson\", \"boettiger-lab/landvote\")\n",
-    "\n",
-    "generate_pmtiles(\"vote_states.geojson\", \"vote_states.pmtiles\")\n",
-    "hf_upload(\"vote_states.pmtiles\", \"boettiger-lab/landvote\")\n",
-    "\n"
+    "generate_pmtiles(\"vote_state.geojson\", \"vote_county.geojson\",\"vote_municipal.geojson\", \"landvote.pmtiles\")\n",
+    "hf_upload(\"landvote.pmtiles\", \"boettiger-lab/landvote\")\n"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "fa397626-6e94-4ab9-a3bb-2bcbd14e8d40",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
+   "cell_type": "markdown",
+   "id": "4af09411-a4fe-475a-8b2b-f2d36388aa19",
+   "metadata": {},
    "source": [
-    "import leafmap.maplibregl as leafmap\n",
-    "m = leafmap.Map(style=\"positron\")\n",
-    "\n",
-    "\n",
-    "url_states = \"https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/vote_states.pmtiles\"\n",
-    "\n",
-    "outcome = [\n",
-    "      'match',\n",
-    "      ['get', 'Status'], \n",
-    "      \"Pass\", '#2E865F',\n",
-    "      \"Fail\", '#FF3300', \n",
-    "      '#ccc'\n",
-    "    ]\n",
-    "paint_states = {\"fill-color\": outcome, \n",
-    "         # \"fill-opacity\": 0.2,\n",
-    "        }\n",
-    "style_states = {\n",
-    "    \"layers\": [\n",
-    "        {\n",
-    "            \"id\": \"votes_states\",\n",
-    "            \"source\": \"vote_states\",\n",
-    "            \"source-layer\": \"vote_states\",\n",
-    "            \"type\": \"fill\",\n",
-    "            \"filter\": [\n",
-    "                \"==\",\n",
-    "                [\"get\", \"year\"],\n",
-    "                2022,\n",
-    "            ],  # only show buildings with height info\n",
-    "            \"paint\": paint_states\n",
-    "        },\n",
-    "    ],\n",
-    "}\n",
-    "\n",
-    "m.add_pmtiles(\n",
-    "    url_states,\n",
-    "    style=style_states,\n",
-    "    visible=True,\n",
-    "    opacity=0.4,\n",
-    "    tooltip=True,\n",
-    "    fit_bounds=False,\n",
-    ")\n",
-    "\n",
-    "m\n",
-    "\n"
+    "# Combine all 3 jurisdiction types into a parquet."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "2f85b6e6-4e50-4602-9c7c-d39f19f99a45",
+   "id": "fa6db898-a132-4ad5-86e9-de80613275d4",
    "metadata": {},
    "outputs": [],
-   "source": []
+   "source": [
+    "df_temp = df_county.union(df_local)\n",
+    "df = df_temp.union(df_state)\n",
+    "df.execute().to_parquet(\"landvote.parquet\")\n",
+    "hf_upload(\"landvote.parquet\", \"boettiger-lab/landvote\")\n"
+   ]
   }
  ],
  "metadata": {