reqs

preprocess.ipynb

@@ -119,7 +119,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.11.10"
   }
  },
  "nbformat": 4,

requirements.txt

@@ -159,7 +159,7 @@ xyzservices==2024.9.0
 yarl==1.9.7
 overturemaps
 tippecanoe
-pandasai
+pandasai==2.3.0
 seaborn
 numpy==1.26.4
 pandas

static-maps.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 26,
    "id": "b313a218-4778-4d5b-9036-f0370d4212a0",
    "metadata": {},
    "outputs": [],
@@ -10,8 +10,6 @@
     "import ibis\n",
     "from ibis import _\n",
     "import streamlit as st\n",
-    "import ibis.expr.datatypes as dt  # Make sure to import the necessary module\n",
-    "\n",
     "\n",
     "conn = ibis.duckdb.connect(extensions=[\"spatial\"])\n",
     "\n",
@@ -29,28 +27,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 27,
    "id": "ba4d8915-cde3-4ef9-ad8c-7759ed2c8a13",
    "metadata": {},
    "outputs": [],
    "source": [
     "vote_county = (votes\n",
-    "                .filter(_[\"Jurisdiction Type\"] == \"County\")\n",
-    "                .rename(county = \"Jurisdiction Name\", state_id = \"State\")\n",
-    "                .mutate(key = _.county + ibis.literal('-') + _.state_id)\n",
-    "                .rename(amount = 'Conservation Funds at Stake', yes = '% Yes')\n",
-    "                .mutate(amount_n=_.amount.replace('$', '').replace(',', '').cast('float'))\n",
-    "                .mutate(log_amount=_.amount_n.log())\n",
-    "                .mutate(year=_['Date'].year().cast('int32'))\n",
-    "                .mutate(\n",
-    "                    yes=ibis.case()\n",
-    "                        .when(_.yes.isin(['Pass', 'None','Fail']), None)  # Handle non-numeric cases\n",
-    "                        .when(_.yes.notnull(), (_.yes.replace('%', '').cast('float').round(2).cast(dt.float64)).cast(dt.string) + '%')  # Convert valid percentages and add %\n",
-    "                        .else_(None)  # Default to None for other cases\n",
-    "                        .end()\n",
-    "                )\n",
-    "               .mutate(log_amount = _.log_amount.round(4))\n",
-    "                .select('key', 'Status', 'yes', 'year', 'amount', 'log_amount', )\n",
+    "               .filter(_[\"Jurisdiction Type\"] == \"County\")\n",
+    "               .rename(county = \"Jurisdiction Name\", state_id = \"State\")\n",
+    "               .mutate(key = _.county + ibis.literal('-') + _.state_id)\n",
+    "               .rename(amount = 'Conservation Funds at Stake', yes = '% Yes')\n",
+    "               .mutate(amount_n=_.amount.replace('$', '').replace(',', '').cast('float'))\n",
+    "               .mutate(log_amount=_.amount_n.log())\n",
+    "               .mutate(year=_['Date'].year().cast('int32'))\n",
+    "               .select('key', 'Status', 'yes', 'year', 'amount', 'log_amount', )\n",
     "               )\n",
     "df_county = (county\n",
     "            .join(states.select(\"state\", \"state_id\"), \"state\")\n",
@@ -65,7 +55,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 28,
    "id": "0cce23c9-245c-4c28-9523-0231eb5acc17",
    "metadata": {},
    "outputs": [],
@@ -78,14 +68,6 @@
     "                .mutate(amount_n=_.amount.replace('$', '').replace(',', '').cast('float'))\n",
     "                .mutate(log_amount=_.amount_n.log())\n",
     "                .mutate(year=_['Date'].year().cast('int32'))\n",
-    "                .mutate(\n",
-    "                    yes=ibis.case()\n",
-    "                        .when(_.yes.isin(['Pass', 'None','Fail']), None)  # Handle non-numeric cases\n",
-    "                        .when(_.yes.notnull(), (_.yes.replace('%', '').cast('float').round(2).cast(dt.float64)).cast(dt.string) + '%')  # Convert valid percentages and add %\n",
-    "                        .else_(None)  # Default to None for other cases\n",
-    "                        .end()\n",
-    "                )\n",
-    "              .mutate(log_amount = _.log_amount.round(4))\n",
     "                .select('key', 'Status', 'yes', 'year', 'amount', 'log_amount', )\n",
     "                )\n",
     "\n",
@@ -96,24 +78,87 @@
     "            .drop('key_right')\n",
     "            .mutate(jurisdiction = ibis.literal(\"Municipal\"))\n",
     "            .cast({\"geometry\": \"geometry\"})\n",
-    "            .mutate(geometry = _.geometry.buffer(.07))\n",
+    "            \n",
     "            )\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 29,
    "id": "a1e81807-8ce3-44bf-9a1c-8563fa33817c",
    "metadata": {},
    "outputs": [],
    "source": [
-    "df = df_county.union(df_local)\n",
-    "# df.execute()"
+    "df = df_county.union(df_local)\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 12,
+   "id": "e0402bc4-9b1b-4d31-8789-1970c34bcfa8",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "47d5f61e2fdc46328e233418e9b48d95",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "gdf = df.execute()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "2d0d5b70-2739-48e8-9ac6-789cb2f9f648",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['State',\n",
+       " 'Jurisdiction Name',\n",
+       " 'Jurisdiction Type',\n",
+       " 'Date',\n",
+       " 'Description',\n",
+       " 'Finance Mechanism',\n",
+       " '\"Other\" Comment',\n",
+       " 'Purpose',\n",
+       " 'Total Funds at Stake',\n",
+       " 'Conservation Funds at Stake',\n",
+       " 'Total Funds Approved',\n",
+       " 'Conservation Funds Approved',\n",
+       " 'Pass?',\n",
+       " 'Status',\n",
+       " '% Yes',\n",
+       " '% No',\n",
+       " 'Notes',\n",
+       " 'Voted Acq. Measure',\n",
+       " 'column18']"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "votes.drop(\"Total Funds )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
    "id": "5d3bee26-7ca8-490c-be5b-fc69a6c3db2a",
    "metadata": {},
    "outputs": [
@@ -123,7 +168,7 @@
      "text": [
       "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n",
       "Token is valid (permission: write).\n",
-      "Your token has been saved to /home/rstudio/.cache/huggingface/token\n",
+      "Your token has been saved to /home/jovyan/.cache/huggingface/token\n",
       "Login successful\n"
      ]
     }
@@ -154,10 +199,10 @@
     "    command = [\n",
     "        \"tippecanoe\",\n",
     "        \"-o\", output_file,\n",
-    "        \"-zg\",\n",
+    "        \"-z\", str(max_zoom),\n",
+    "        \"--drop-densest-as-needed\",\n",
     "        \"--extend-zooms-if-still-dropping\",\n",
     "        \"--force\",\n",
-    "        \"--projection\", \"EPSG:4326\",  \n",
     "        input_file\n",
     "    ]\n",
     "\n",
@@ -172,49 +217,88 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
-   "id": "902ece1d-6e57-4f83-9286-709dbb549fae",
+   "execution_count": null,
+   "id": "ac91a627-70a8-4e60-b3ef-66e9c1e02762",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.execute().to_file(\"vote.geojson\")\n",
+    "generate_pmtiles(\"vote.geojson\", \"vote.pmtiles\")\n",
+    "hf_upload(\"vote.pmtiles\", \"boettiger-lab/landvote\")\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "a3bde171-e7a8-4a5d-97ea-bfffdf26918b",
    "metadata": {},
    "outputs": [
     {
      "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "2b18fcc2775845c4a3b40eedbe32475b",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "c7314c0c7d9948c9a3feaddadccde55b",
-       "version_major": 2,
-       "version_minor": 0
-      },
+      "text/html": [
+       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">DatabaseTable: ibis_read_geo_pamy33s53vaepploan6xzko2mu\n",
+       "  key          string\n",
+       "  Status       string\n",
+       "  yes          string\n",
+       "  year         int32\n",
+       "  amount       string\n",
+       "  log_amount   float64\n",
+       "  jurisdiction string\n",
+       "  geom         geospatial:geometry\n",
+       "</pre>\n"
+      ],
       "text/plain": [
-       "vote.geojson:   0%|          | 0.00/92.9M [00:00<?, ?B/s]"
+       "DatabaseTable: ibis_read_geo_pamy33s53vaepploan6xzko2mu\n",
+       "  key          string\n",
+       "  Status       string\n",
+       "  yes          string\n",
+       "  year         int32\n",
+       "  amount       string\n",
+       "  log_amount   float64\n",
+       "  jurisdiction string\n",
+       "  geom         geospatial:geometry"
       ]
      },
+     "execution_count": 25,
      "metadata": {},
-     "output_type": "display_data"
-    },
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import leafmap.maplibregl as leafmap\n",
+    "import geopandas\n",
+    "url = \"https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/vote.geojson\"\n",
+    "\n",
+    "conn.read_geo(url).filter\n",
+    "#gpf = geopandas.read_file(url, engine=\"pyogrio\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "58094159-4efc-4b56-a1d4-dac27be86924",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gpf.to_file(\"vote.geojson\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "221db74d-961c-4cac-a443-02b3d562b531",
+   "metadata": {},
+   "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
       "For layer 0, using name \"vote\"\n",
-      "vote.geojson:496: null geometry (additional not reported): in JSON object {\"type\":\"Feature\",\"properties\":{\"key\":null,\"Status\":\"Fail\",\"yes\":\"43.67%\",\"year\":1990,\"amount\":null,\"log_amount\":null,\"jurisdiction\":\"County\"},\"geometry\":null}\n",
-      "2195 features, 10746823 bytes of geometry and attributes, 58352 bytes of string pool, 0 bytes of vertices, 0 bytes of nodes\n",
-      "Choosing a maxzoom of -z3 for features typically 81840 feet (24945 meters) apart, and at least 10982 feet (3348 meters) apart\n",
-      "Choosing a maxzoom of -z11 for resolution of about 214 feet (65 meters) within features\n",
-      "  99.9%  11/189/562  \n",
-      "  100.0%  11/110/897  \r"
+      "vote.geojson:496: null geometry (additional not reported): in JSON object {\"type\":\"Feature\",\"properties\":{\"key\":null,\"Status\":\"Fail\",\"yes\":\"43.668463401297%\",\"year\":1990,\"amount\":null,\"log_amount\":null,\"jurisdiction\":\"County\"},\"geometry\":null}\n",
+      "2195 features, 10372266 bytes of geometry, 95306 bytes of string pool\n",
+      "  99.9%  12/1136/1649  \n",
+      "  100.0%  12/220/1795  \r"
      ]
     },
     {
@@ -223,66 +307,64 @@
      "text": [
       "Successfully generated PMTiles file: vote.pmtiles\n"
      ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "1176d2d9c19e45539fc9efc2a60db944",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "vote.pmtiles:   0%|          | 0.00/4.81M [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
     }
    ],
    "source": [
-    "gdf= df.execute()\n",
-    "gdf = gdf.set_crs(\"EPSG:4326\")\n",
-    "# print(gdf.crs)\n",
-    "# gdf.to_parquet(\"votes.parquet\")\n",
-    "\n",
-    "gdf.to_file(\"vote.geojson\")\n",
-    "hf_upload(\"vote.geojson\", \"boettiger-lab/landvote\")\n",
-    "\n",
-    "generate_pmtiles(\"vote.geojson\", \"vote.pmtiles\")\n",
-    "hf_upload(\"vote.pmtiles\", \"boettiger-lab/landvote\")\n",
-    "\n"
+    "generate_pmtiles(\"vote.geojson\", \"vote.pmtiles\")\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
-   "id": "fa397626-6e94-4ab9-a3bb-2bcbd14e8d40",
-   "metadata": {
-    "scrolled": true
-   },
+   "execution_count": 22,
+   "id": "23deeb8c-3ef9-4279-bd5f-7b72494ee567",
+   "metadata": {},
    "outputs": [
     {
      "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d829e5753c414cc2887dac52fd35dc82",
-       "version_major": 2,
-       "version_minor": 1
-      },
       "text/plain": [
-       "Map(height='600px', map_options={'bearing': 0, 'center': (0, 20), 'pitch': 0, 'style': 'https://basemaps.carto…"
+       "<Projected CRS: EPSG:3857>\n",
+       "Name: WGS 84 / Pseudo-Mercator\n",
+       "Axis Info [cartesian]:\n",
+       "- X[east]: Easting (metre)\n",
+       "- Y[north]: Northing (metre)\n",
+       "Area of Use:\n",
+       "- name: World between 85.06°S and 85.06°N.\n",
+       "- bounds: (-180.0, -85.06, 180.0, 85.06)\n",
+       "Coordinate Operation:\n",
+       "- name: Popular Visualisation Pseudo-Mercator\n",
+       "- method: Popular Visualisation Pseudo Mercator\n",
+       "Datum: World Geodetic System 1984 ensemble\n",
+       "- Ellipsoid: WGS 84\n",
+       "- Prime Meridian: Greenwich"
       ]
      },
-     "execution_count": 15,
+     "execution_count": 22,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
+   "source": [
+    "gdf = geopandas.read_file(\"vote.pmtiles\", engine=\"pyogrio\")\n",
+    "gdf.crs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fa397626-6e94-4ab9-a3bb-2bcbd14e8d40",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
    "source": [
     "import leafmap.maplibregl as leafmap\n",
     "m = leafmap.Map(style=\"positron\")\n",
     "\n",
     "url = \"https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/vote.pmtiles\"\n",
     "\n",
+    "#gdf = df.filter(_.year==1988).execute()\n",
+    "#gdf.to_file(\"vote.geojson\")\n",
+    "\n",
     "outcome = [\n",
     "      'match',\n",
     "      ['get', 'Status'], \n",
@@ -304,7 +386,7 @@
     "            \"filter\": [\n",
     "                \"==\",\n",
     "                [\"get\", \"year\"],\n",
-    "                2022,\n",
+    "                1988,\n",
     "            ],  # only show buildings with height info\n",
     "            \"paint\": paint\n",
     "        },\n",
@@ -319,15 +401,20 @@
     "    tooltip=True,\n",
     "    fit_bounds=False,\n",
     ")\n",
-    "\n",
+    "#m.add_layer_control()\n",
     "m\n",
-    "\n"
+    "\n",
+    "\n",
+    "\n",
+    "#m.add_geojson(\"vote.geojson\", \"fill-extrusion\", paint = paint)\n",
+    "#m.add_gdf(gdf, \"fill-extrusion\", paint = paint)\n",
+    "#m"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "2f85b6e6-4e50-4602-9c7c-d39f19f99a45",
+   "id": "5e521f00-1b04-4016-9a6a-71a12e846dd3",
    "metadata": {},
    "outputs": [],
    "source": []
@@ -349,7 +436,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.11.10"
   }
  },
  "nbformat": 4,