cleaning up code

app.py

@@ -1,10 +1,15 @@
 import ibis
 from ibis import _
-import streamlit as st
 
+import streamlit as st
 import altair as alt
-
-
+import os
+import pandas as pd
+import matplotlib.pyplot as plt
+from pandasai.llm.openai import OpenAI
+from pandasai import Agent
+from pandasai.responses.streamlit_response import StreamlitResponse
+import leafmap.maplibregl as leafmap
 
 st.set_page_config(layout="wide",
                    page_title="TPL LandVote",
@@ -17,32 +22,21 @@ An experimental platform for visualizing data on ballot measures for conservatio
 
 '''
 
-dark_orange = 'rgba(171, 86, 1, 1)'   # dark orange - min value 
-light_orange = 'rgba(243, 211, 177, 1)' # light orange 
-grey = 'rgba(211, 211, 211, 1)'        # grey
-light_green = 'rgba(195, 219, 195, 1)' # light green 
-dark_green = 'rgba(65, 125, 65, 1)'    # dark green  - max value 
+COLORS = {
+    "dark_orange": "#ab5601",
+    "light_orange": "#f3d3b1",
+    "grey": "#d3d3d3",
+    "light_green": "#c3dbc3",
+    "dark_green": "#417d41",
+    "dem_blue": "#1b46c2",
+    "rep_red": "#E81B23"
+}
 
-dem_blue = "#1b46c2"
-rep_red = "#E81B23"
-
-## Chatbot
-import os
-import pandas as pd
-import matplotlib.pyplot as plt
-from pandasai.llm.openai import OpenAI
-from pandasai import Agent
-from pandasai.responses.streamlit_response import StreamlitResponse
 
+## chatbot
 llm = OpenAI(api_token=st.secrets["OPENAI_API_KEY"])
 df1 = pd.read_csv("data.csv")
-
-agent = Agent(
-    [df1],
-    config={"verbose": True, "response_parser": StreamlitResponse, "llm": llm},
-)
-
-
+agent = Agent([df1], config={"verbose": True, "response_parser": StreamlitResponse, "llm": llm})
 
 with st.sidebar:
 
@@ -67,178 +61,132 @@ with st.sidebar:
                 st.image(im)
                 os.remove('exports/charts/temp_chart.png')
             st.write(resp)
-        
 
+year = st.slider("Select a year", 1988, 2024, 2022, 1)
 
-        
-# year = st.slider("Select a year", min_value=1988, max_value=2024, value=2022, step=2)
-year = st.slider("Select a year", min_value=1988, max_value=2024, value=2022, step=1)
+ 
 
+votes_pmtiles = "https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/votes.pmtiles"
+votes_parquet = "https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/votes.parquet"
 
-url = "https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/votes.pmtiles"
-parties = "https://huggingface.co/datasets/boettiger-lab/landvote/resolve/main/votes.parquet"
-
+# get parquet data for charts
 con = ibis.duckdb.connect(extensions=["spatial"])
-
 party = (con
-            .read_parquet(parties)
-            .cast({"geometry": "geometry"})
-            )
-
-def get_passes(party):
-    df = (party
-        .filter(_.year >= 2000)
-        .group_by("year", "party")
-        .aggregate(total=_.count(),
-                   passes = (_.Status.isin(["Pass", "Pass*"]).sum())
-                  )
-        .mutate(
-            percent_passed=(_.passes / _.total).round(2),
-            color=ibis.case()
-                .when(_.party == "DEMOCRAT", ibis.literal("#2e4a93"))
-                .else_(ibis.literal("#E81B23"))
-                .end()
+         .read_parquet(votes_parquet)
+         .cast({"geometry": "geometry"})
         )
-    )
-    
-    df = df.to_pandas()
-    return df
 
 
-def percent_chart(df_passes):
-    alt.themes.enable('fivethirtyeight')
+# generate altair chart with df
+def create_chart(df, y_column, title, color):
+    # color encoding - color is a list or single value 
+    color_encoding = (
+        alt.Color('party:N', scale=alt.Scale(domain=["DEMOCRAT", "REPUBLICAN"], range=color))
+        if isinstance(color, list) else alt.value(color)
+    )
 
-    chart = alt.Chart(df_passes).mark_line(strokeWidth=3).encode(
-    x=alt.X('year:N', title='Year'),
-    y=alt.Y('percent_passed:Q', title='Percent Passed'),
-    color=alt.Color('party:N',  # Map 'party' to color
-                    scale=alt.Scale(domain=["DEMOCRAT", "REPUBLICAN"], 
-                                    range=["#1b46c2", "#E81B23"]),
-                    legend=alt.Legend(title="Party")
-                   )
+    return alt.Chart(df).mark_line(strokeWidth=3).encode(
+        x=alt.X('year:N', title='Year'),
+        y=alt.Y(f'{y_column}:Q', title=title),
+        color=color_encoding  
     ).properties(
-        title='% of Measures Passed'
+        title=title
     )
-    
-    st.altair_chart(chart, use_container_width=True)
+
+# percentage of measures passing, per party
+def get_passes(party):
+    return (party
+        .filter(_.year >= 2000)
+        .group_by("year", "party")
+        .aggregate(total=_.count(), passes=_.Status.isin(["Pass", "Pass*"]).sum())
+        .mutate(percent_passed=(_.passes / _.total).round(2),
+                color=ibis.case().when(_.party == "DEMOCRAT", COLORS["dem_blue"]).else_(COLORS["rep_red"]).end())
+        .to_pandas())
 
 
+# cumulative funding over time 
 def funding_chart(party):
-    df = (party
-        .mutate(amount=_.amount.replace('$', '').replace(',', '').cast('float64'))
-        .filter(_.Status.isin(["Pass", "Pass*"]))
-        .group_by("year")
-        .aggregate(total_funding=_.amount.sum())
-        .order_by("year")
-        .mutate(cumulative_funding=_.total_funding.cumsum()/1e9)
-        .execute()
-        )
-  
-    chart = alt.Chart(df).mark_line(strokeWidth=3).encode(
-        x=alt.X('year:N', title='Year'),
-        y=alt.Y('cumulative_funding:Q', title='Billions of Dollars'),
-    ).properties(
-        title='Cumulative Funding'
-    )
+   return (party
+           .filter(_.year >= 2000)
+          .mutate(amount=_.amount.replace('$', '')
+                  .replace(',', '')
+                  .cast('float64'))
+          .filter(_.Status.isin(["Pass", "Pass*"]))
+          .group_by("year")
+          .aggregate(total_funding=_.amount.sum())
+          .order_by("year")
+          .mutate(cumulative_funding=_.total_funding.cumsum()/1e9)
+          .to_pandas()
+         )
     
-    st.altair_chart(chart, use_container_width=True)
-
 
+#color fill for measure status
 paint_fill = {
-                "fill-color": [
-                    "case",
-                    # if passed, color green
-                    ["==", ["get", "Status"], "Pass"],
-                    [
-                        "interpolate", ["linear"], [
-                            "to-number", ["slice", ["get", "yes"], 0, -1]  # convert 'yes' string to number
-                        ],
-                        50, grey,    
-                        55, light_green,     # higher yes % -> darker green
-                        100, dark_green     # 100 is the max of data 
-                    ],
-                    # if failed, color orange
-                    ["==", ["get", "Status"], "Fail"],
-                    [
-                        "interpolate", ["linear"], [
-                            "to-number", ["slice", ["get", "yes"], 0, -1]  # convert 'yes' string to number
-                        ],
-                        0, dark_orange,     # higher yes % -> lighter orange 
-                        50, light_orange,   
-                        67, grey # 67 is the max of data.
-                    ],
-                    grey  # if no match 
-                ]
-            }
-
+    "fill-color": [
+        "case",
+        ["==", ["get", "Status"], "Pass"],
+        [
+            "interpolate", ["linear"], [
+                "to-number", ["slice", ["get", "yes"], 0, -1]  # convert 'yes' string to number
+            ],
+            50, COLORS["grey"],
+            55, COLORS["light_green"],
+            100, COLORS["dark_green"] # higher yes % -> darker green
+        ],
+        ["==", ["get", "Status"], "Fail"],
+        [
+            "interpolate", ["linear"], [
+                "to-number", ["slice", ["get", "yes"], 0, -1]
+            ],
+            0, COLORS["dark_orange"],
+            50, COLORS["light_orange"], # lower yes % -> darker orange
+            67, COLORS["grey"] # 67 is max in our data 
+        ],
+        COLORS["grey"]
+    ]
+}
 
+# for status, height depends on funding 
 paint_extrusion = {
-                "fill-extrusion-color": [
-                    "case",
-                    # if passed, color green
-                    ["==", ["get", "Status"], "Pass"],
-                    [
-                        "interpolate", ["linear"], [
-                            "to-number", ["slice", ["get", "yes"], 0, -1]  # convert 'yes' string to number
-                        ],
-                        50, grey,    
-                        55, light_green,     # higher yes % -> darker green
-                        100, dark_green     # 100 is the max of data 
-                    ],
-                    # if failed, color orange
-                    ["==", ["get", "Status"], "Fail"],
-                    [
-                        "interpolate", ["linear"], [
-                            "to-number", ["slice", ["get", "yes"], 0, -1]  # convert 'yes' string to number
-                        ],
-                        0, dark_orange,     # higher yes % -> lighter orange 
-                        50, light_orange,   
-                        67, grey # 67 is the max of data.
-                    ],
-                    grey  # if no match 
-                ],
-                "fill-extrusion-height": ["*", ["get", "log_amount"], 5000],
-            }
+    "fill-extrusion-color": paint_fill["fill-color"],
+    "fill-extrusion-height": ["*", ["to-number", ["get", "log_amount"]], 5000]
+}
 
 
+
+# pmtiles style for status 
 def get_style_status(jurisdiction):
     if jurisdiction == "State":
         name = "state"
         label = "states"
         paint_type = paint_fill
         layer_type = "fill"
-
     elif jurisdiction == "County":
         name = "county"
         label = "counties"
         paint_type = paint_extrusion
         layer_type = "fill-extrusion"
-
-    else: 
+    else:  # Municipal
         name = "municipal"
         label = "cities"
         paint_type = paint_extrusion
         layer_type = "fill-extrusion"
 
-    style = {
+    return {
         "layers": [
             {
                 "id": label,
                 "source": name,
                 "source-layer": name,
                 "type": layer_type,
-                "filter": [
-                    "==",
-                    ["get", "year"],
-                    year,
-                ],
+                "filter": ["==", ["get", "year"], year],
                 "paint": paint_type
-            },
-        ],
+            }
+        ]
     }
-    return style 
 
 
+# pmtiles style for party 
 def get_style_party(jurisdiction):
     if jurisdiction == "State":
         name = "state"
@@ -246,119 +194,57 @@ def get_style_party(jurisdiction):
     elif jurisdiction == "County":
         name = "county"
         label = "counties"
-
-    else: 
+    else:  # Municipal
         name = "municipal"
         label = "cities"
 
-
-    style_party = {
+    # Return style dictionary for political party
+    return {
         "layers": [
             {
-                "id": label,
+                "id": label, 
                 "source": name,
                 "source-layer": name,
                 "type": "fill",
                 "filter": [
-                    "==",
-                    ["get", "year"],
-                    year,
+                    "==", ["get", "year"], year
                 ],
                 "paint": {
-                "fill-color": 
-                    {
-                    'property': 'party',
-                    'type': 'categorical',
-                    'stops': [
-                        ["DEMOCRAT", dem_blue],
-                        ["REPUBLICAN", rep_red],
-                    ]
+                    "fill-color": {
+                        "property": "party",
+                        "type": "categorical",
+                        "stops": [
+                            ["DEMOCRAT", COLORS["dem_blue"]],
+                            ["REPUBLICAN", COLORS["rep_red"]]
+                        ]
                     }
                 }
-            },
-        ],
+            }
+        ]
     }
-    return style_party
-
 
 
-style_options= ["Measure Status", "Political Party"]
-
-color_choice = st.radio("Color by:", style_options)
-            
-import leafmap.maplibregl as leafmap
-m = leafmap.Map(style="positron",   center=(-100, 40), zoom=3)
+m = leafmap.Map(style="positron", center=(-100, 40), zoom=3)
+color_choice = st.radio("Color by:", ["Measure Status", "Political Party"])
 
 if color_choice == "Measure Status":
+    m.add_pmtiles(votes_pmtiles, style=get_style_status("State"), visible=True, opacity=0.4, tooltip=True)
+    m.add_pmtiles(votes_pmtiles, style=get_style_status("County"), visible=True, opacity=1.0, tooltip=True)
+    m.add_pmtiles(votes_pmtiles, style=get_style_status("Municipal"), visible=True, opacity=1.0, tooltip=True)
 
-        #states are 2D and transparent, thus added separately.
-    m.add_pmtiles(
-        url,
-        style=get_style_status("State"),
-        visible=True,
-        opacity=0.4,
-        tooltip=True,
-        fit_bounds=False
-    )
-    
-    m.add_pmtiles(
-        url,
-        style=get_style_status("County"),
-        visible=True,
-        opacity=1.0,
-        tooltip=True,
-        fit_bounds=False
-    )
-    
-    
-    m.add_pmtiles(
-        url,
-        style=get_style_status("Municipal"),
-        visible=True,
-        opacity=1.0,
-        tooltip=True,
-        fit_bounds=False
-    )
-    
-else:
-    m.add_pmtiles(
-    url,
-    style=get_style_party("State"),
-    visible=True,
-    opacity=0.6,
-    tooltip=True,
-    fit_bounds=False
-    )
-    
-    m.add_pmtiles(
-        url,
-        style=get_style_party("County"),
-        visible=True,
-        opacity=1.0,
-        tooltip=True,
-        fit_bounds=False
-    )
-    
-    
-    m.add_pmtiles(
-        url,
-        style=get_style_party("Municipal"),
-        visible=True,
-        opacity=1.0,
-        tooltip=True,
-        fit_bounds=False
-    )
-    
-    
-m.add_layer_control()
+elif color_choice == "Political Party":
+    m.add_pmtiles(votes_pmtiles, style=get_style_party("State"), visible=True, opacity=0.6, tooltip=True)
+    m.add_pmtiles(votes_pmtiles, style=get_style_party("County"), visible=True, opacity=1.0, tooltip=True)
+    m.add_pmtiles(votes_pmtiles, style=get_style_party("Municipal"), visible=True, opacity=1.0, tooltip=True)
 
+#
+m.add_layer_control()
 m.to_streamlit()
 
-df_passes = get_passes(party)
-percent_chart(df_passes)
-funding_chart(party.filter(_.year >= 2000))
 
+# display charts
+df_passes = get_passes(party)
+st.altair_chart(create_chart(df_passes, "percent_passed", "% of Measures Passed", [COLORS["dem_blue"], COLORS["rep_red"]]), use_container_width=True)
 
-# st.divider()
-# footer = st.container()
-
+df_funding = funding_chart(party)
+st.altair_chart(create_chart(df_funding, "cumulative_funding", "Cumulative Funding (Billions of Dollars)", COLORS["dark_green"]))

preprocess.ipynb

@@ -10,7 +10,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "6452373c-d10f-472c-9174-cd05a3363587",
    "metadata": {},
    "outputs": [],
@@ -30,7 +30,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "dd9ffc64-b4cf-4e5b-9c96-703e91a77837",
    "metadata": {},
    "outputs": [],
@@ -74,7 +74,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "8fc40e96-fffd-4b23-9963-c931fdce96f6",
    "metadata": {},
    "outputs": [],
@@ -105,21 +105,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "b1cd8f44-57fa-49a8-b438-f9e4aab747c5",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n",
-      "Token is valid (permission: write).\n",
-      "Your token has been saved to /home/rstudio/.cache/huggingface/token\n",
-      "Login successful\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import subprocess\n",
     "import os\n",
@@ -165,85 +154,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "7061577e-0632-4395-8ac5-241a1fab53b0",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ff135769c16340f9a87905b600836afe",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "037e0563f82447998bbd438473a2017d",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "60716409890f4760a744123d4e6003de",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2286 features, 17891307 bytes of geometry and attributes, 58357 bytes of string pool, 0 bytes of vertices, 0 bytes of nodes\n",
-      "Choosing a maxzoom of -z3 for features typically 83927 feet (25581 meters) apart, and at least 11120 feet (3390 meters) apart\n",
-      "Choosing a maxzoom of -z10 for resolution of about 278 feet (84 meters) within features\n",
-      "  99.9%  10/279/424  \n",
-      "  100.0%  10/55/448  \r"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Successfully generated PMTiles file: votes.pmtiles\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "c8a1a66665814dd8b0ad839929a6b30d",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "votes.pmtiles:   0%|          | 0.00/4.72M [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "gdf_state = votes.filter(_.jurisdiction == 'State').execute().set_crs(\"EPSG:4326\")\n",
     "gdf_state.to_file(\"votes_state.geojson\")\n",
@@ -260,39 +174,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "id": "f2979624-bcdf-4a8a-899a-c22fc3cdaf0e",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "6b928b70b56e45a8b1f59c727bd6879a",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "0f4fa7e221ff481697375deba7f34b94",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "votes.parquet:   0%|          | 0.00/43.1M [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "# save as parquet\n",
     "votes.execute().set_crs(\"EPSG:4326\").to_parquet(\"votes.parquet\")\n",