maybe got it

explore.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 16,
    "id": "f7e6298c-d886-432a-a1b7-c3fee914c24f",
    "metadata": {},
    "outputs": [],
@@ -20,26 +20,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 104,
+   "execution_count": null,
    "id": "bd9c2561-334a-40d6-adf0-ce20a991a9a7",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "f46fb7de69784a4e89763c5432d06a40",
-       "version_major": 2,
-       "version_minor": 1
-      },
-      "text/plain": [
-       "Map(height='600px', map_options={'bearing': 0, 'center': (0, 20), 'pitch': 0, 'style': 'https://basemaps.carto…"
-      ]
-     },
-     "execution_count": 104,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "# Visualize the full data to show overlap:\n",
     "import leafmap.maplibregl as leafmap\n",
@@ -76,7 +60,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 158,
+   "execution_count": 10,
    "id": "3ceed4c7-b7a2-436e-bca1-c57e7133cdf6",
    "metadata": {},
    "outputs": [],
@@ -84,9 +68,8 @@
     "# Make things faster.  Let's zoom in on a subset of the data. \n",
     "\n",
     "from ibis.interactive import *\n",
-    "\n",
     "ca_parquet = \"/home/rstudio/source.coop/cboettig/ca30x30/ca_areas.parquet\"\n",
-    "con = ibis.duckdb.connect()\n",
+    "conn = ibis.duckdb.connect()\n",
     "\n",
     "# Ugly code for a cookie cutter zoom\n",
     "# Turns out we could have just used filter(_.UNIT_NAME == \"Angeles National Forest\")\n",
@@ -98,19 +81,41 @@
     "cookie_cutter = ibis.literal(gdf.geometry.iloc[0])\n",
     "\n",
     "angeles_forest = (\n",
-    "    con.read_parquet(ca_parquet)\n",
+    "    conn.read_parquet(ca_parquet)\n",
     "    .cast({\"SHAPE\": \"geometry\"})\n",
-    "    .mutate(geom = _.SHAPE.convert(\"epsg:3310\",\"epsg:4326\"))\n",
-    "    .filter( _.geom.intersects(cookie_cutter))\n",
+    "    .rename(geom = \"SHAPE\")\n",
+    " #   .mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\"))\n",
+    " #   .filter( _.geom.intersects(cookie_cutter))\n",
     "    .filter(_.UNIT_NAME == \"Angeles National Forest\") \n",
     "    .select(\"reGAP\", \"UNIT_NAME\", \"MNG_AGNCY\", \"Release_Year\", \"OBJECTID\", \"geom\")\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a0cb34b1-8d70-49bf-80c6-244ecc8ddf84",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import ibis\n",
+    "from ibis import _\n",
+    "\n",
+    "tbl = (\n",
+    "    conn.read_parquet(\"https://data.source.coop/cboettig/ca30x30/ca_areas.parquet\")\n",
+    "    .cast({\"SHAPE\": \"geometry\"})\n",
+    "    .filter(_.UNIT_NAME == \"Angeles National Forest\", _.reGAP < 3) \n",
     ")\n",
-    "\n"
+    "\n",
+    "tbl_2023 = tbl.filter(_.Release_Year == 2023)\n",
+    "tbl_2024 = tbl.filter(_.Release_Year == 2024)\n",
+    "intersects = tbl_2024.anti_join(tbl_2023, _.SHAPE.intersects(tbl_2023.SHAPE))\n",
+    "intersects"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 159,
+   "execution_count": 11,
    "id": "3ed21429-a75e-4b52-b6eb-d41c8f051883",
    "metadata": {},
    "outputs": [],
@@ -122,7 +127,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 160,
+   "execution_count": 12,
    "id": "7bfac88a-822a-4e4e-883e-b7e51544abd1",
    "metadata": {},
    "outputs": [],
@@ -133,21 +138,23 @@
     "\n",
     "# THIS IS ALMOST WHAT WE WANT, BUT for some issues\n",
     "# All reGAP 1,2 polygons in 2024 that don't intersect 2023\n",
-    "la_intersects = la_2024.anti_join(la_2023, _.geom.intersects(la_2023.geom))\n"
+    "la_intersects = la_2024.anti_join(la_2023, _.geom.intersects(la_2023.geom))\n",
+    "\n",
+    "#la_contains = la_2024.anti_join(la_2023, _.geom.contains(la_2023.geom.centroid()))\n",
+    "#centroids = la_2023.filter(_.reGAP == 2).mutate( geom = la_2023.filter(_.reGAP == 2).geom.centroid() )\n",
+    "#centroids = old.mutate( geom = old.geom.centroid() )"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 164,
+   "execution_count": 23,
    "id": "ddae1f9e-988a-4aed-bc71-f1c40d2bb0ea",
-   "metadata": {
-    "scrolled": true
-   },
+   "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "52b3b9c8cb1b4bbf8fe16aebef4d2e25",
+       "model_id": "ff9365d5ed2848399f1a00952cf6508f",
        "version_major": 2,
        "version_minor": 1
       },
@@ -155,7 +162,7 @@
        "Map(height='600px', map_options={'bearing': 0, 'center': (0, 20), 'pitch': 0, 'style': 'https://basemaps.carto…"
       ]
      },
-     "execution_count": 164,
+     "execution_count": 23,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -179,51 +186,21 @@
     "paint = {\"fill-color\": gap, \"fill-opacity\": 0.5}\n",
     "\n",
     "m = leafmap.Map(style=\"positron\")\n",
-    "m.add_gdf(allgap_2023.execute(),layer_type=\"fill\", name = \"allgap_2023\", paint=paint)\n",
-    "m.add_gdf(allgap_2024.execute(),layer_type=\"fill\", name = \"allgap_2024\", paint=paint)\n",
-    "#m.add_gdf(la_2023.execute(),layer_type=\"fill\", name = \"2023\", paint=paint)\n",
-    "#m.add_gdf(la_2024.execute(),layer_type=\"fill\", name = \"2024\", paint=paint)\n",
-    "m.add_gdf(la_intersects.execute(),layer_type=\"fill\", name = \"anti_intersects\", paint=paint)\n",
-    "#m.add_gdf(intersects.execute(), layer_type=\"fill\", name = \"overlapping\", paint=paint)\n",
+    "#m.add_gdf(allgap_2023.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"allgap_2023\", paint=paint)\n",
+    "#m.add_gdf(allgap_2024.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"allgap_2024\", paint=paint)\n",
+    "m.add_gdf(la_2023.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"2023\", paint=paint)\n",
+    "#m.add_gdf(la_2024.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"2024\", paint=paint)\n",
+    "#m.add_gdf(la_intersects.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"anti_intersects\", paint=paint)\n",
     "m.add_layer_control()\n",
     "m"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 170,
+   "execution_count": null,
    "id": "6832038f-0342-495f-8cbf-9a8f3cc83aac",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┓\n",
-       "┃<span style=\"font-weight: bold\"> OBJECTID_right </span>┃<span style=\"font-weight: bold\"> geom                                                                             </span>┃<span style=\"font-weight: bold\"> reGAP </span>┃\n",
-       "┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━┩\n",
-       "│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int64</span>          │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">geospatial:geometry</span>                                                              │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int16</span> │\n",
-       "├────────────────┼──────────────────────────────────────────────────────────────────────────────────┼───────┤\n",
-       "│          <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">21105</span> │ <span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">MULTIPOLYGON</span><span style=\"color: #000000; text-decoration-color: #000000\"> Z </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">(((</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-118.017</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34.499</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-118.019</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34.499</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-118.019</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34.499</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #808000; text-decoration-color: #808000\">...</span><span style=\"font-weight: bold\">&gt;</span> │     <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> │\n",
-       "│          <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">22618</span> │ <span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">MULTIPOLYGON</span><span style=\"color: #000000; text-decoration-color: #000000\"> Z </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">(((</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-118.297</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34.433</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-118.298</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34.433</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-118.303</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34.433</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #808000; text-decoration-color: #808000\">...</span><span style=\"font-weight: bold\">&gt;</span> │     <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> │\n",
-       "└────────────────┴──────────────────────────────────────────────────────────────────────────────────┴───────┘\n",
-       "</pre>\n"
-      ],
-      "text/plain": [
-       "┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┓\n",
-       "┃\u001b[1m \u001b[0m\u001b[1mOBJECTID_right\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mgeom\u001b[0m\u001b[1m                                                                            \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mreGAP\u001b[0m\u001b[1m \u001b[0m┃\n",
-       "┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━┩\n",
-       "│ \u001b[2mint64\u001b[0m          │ \u001b[2mgeospatial:geometry\u001b[0m                                                              │ \u001b[2mint16\u001b[0m │\n",
-       "├────────────────┼──────────────────────────────────────────────────────────────────────────────────┼───────┤\n",
-       "│          \u001b[1;36m21105\u001b[0m │ \u001b[1m<\u001b[0m\u001b[1;95mMULTIPOLYGON\u001b[0m\u001b[39m Z \u001b[0m\u001b[1;39m(\u001b[0m\u001b[1;39m(\u001b[0m\u001b[1;39m(\u001b[0m\u001b[1;36m-118.017\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m34.499\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m0\u001b[0m\u001b[39m, \u001b[0m\u001b[1;36m-118.019\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m34.499\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m0\u001b[0m\u001b[39m, \u001b[0m\u001b[1;36m-118.019\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m34.499\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m0\u001b[0m\u001b[39m, \u001b[0m\u001b[33m...\u001b[0m\u001b[1m>\u001b[0m │     \u001b[1;36m2\u001b[0m │\n",
-       "��          \u001b[1;36m22618\u001b[0m │ \u001b[1m<\u001b[0m\u001b[1;95mMULTIPOLYGON\u001b[0m\u001b[39m Z \u001b[0m\u001b[1;39m(\u001b[0m\u001b[1;39m(\u001b[0m\u001b[1;39m(\u001b[0m\u001b[1;36m-118.297\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m34.433\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m0\u001b[0m\u001b[39m, \u001b[0m\u001b[1;36m-118.298\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m34.433\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m0\u001b[0m\u001b[39m, \u001b[0m\u001b[1;36m-118.303\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m34.433\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m0\u001b[0m\u001b[39m, \u001b[0m\u001b[33m...\u001b[0m\u001b[1m>\u001b[0m │     \u001b[1;36m2\u001b[0m │\n",
-       "└────────────────┴──────────────────────────────────────────────────────────────────────────────────┴───────┘"
-      ]
-     },
-     "execution_count": 170,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "## It intersects these, even though it looks like it shouldn't\n",
     "\n",
@@ -239,14 +216,97 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 162,
+   "execution_count": null,
    "id": "b41b0af6-635c-4ffb-9f19-342284f18e62",
    "metadata": {},
+   "outputs": [],
+   "source": [
+    "import leafmap.maplibregl as leafmap\n",
+    "import ibis\n",
+    "from ibis import _\n",
+    "## Minimal look\n",
+    "areas = (\n",
+    "    conn.read_parquet(\"https://data.source.coop/cboettig/ca30x30/ca_areas.parquet\")\n",
+    "    .cast({\"SHAPE\": \"geometry\"})\n",
+    "    .rename(geom = \"SHAPE\")\n",
+    "    .select(\"Release_Year\", \"reGAP\", \"OBJECTID\", \"geom\")\n",
+    "    .mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\"))\n",
+    "    \n",
+    ")\n",
+    "new = areas.filter(_.OBJECTID == 111913)\n",
+    "old = areas.filter(_.OBJECTID == 21105)\n",
+    "# old.geom.difference(new.geom)\n",
+    "\n",
+    "\n",
+    "m = leafmap.Map(style=\"positron\")\n",
+    "m.add_gdf(old.execute(),layer_type=\"fill\", name = \"2023\", paint=paint)\n",
+    "m.add_gdf(new.execute(),layer_type=\"fill\", name = \"2024\", paint=paint)\n",
+    "m.add_layer_control()\n",
+    "m"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ab985b4f-2257-46ce-95af-508c17323ca3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "96cefbe6-12c9-4ca4-9352-c3573924c91c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "la_2023.execute().to_file(\"la_2023.geojson\")\n",
+    "la_2024.execute().to_file(\"la_2024.geojson\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4b9ee360-8583-4096-b1e1-43772f32a182",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "la_2024.filter(_.OBJECTID == 111913).execute().to_file(\"new.geojson\")\n",
+    "la_2023.filter(_.OBJECTID == 21105).execute().to_file(\"old.geojson\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "id": "1e538620-19f5-42eb-8233-e280b5c5b920",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import ibis\n",
+    "from ibis import _\n",
+    "\n",
+    "tbl = (\n",
+    "    conn.read_parquet(\"https://data.source.coop/cboettig/ca30x30/ca_areas.parquet\")\n",
+    "    .cast({\"SHAPE\": \"geometry\"})\n",
+    "    .filter(_.UNIT_NAME == \"Angeles National Forest\", _.reGAP < 3) \n",
+    ")\n",
+    "\n",
+    "tbl_2023 = tbl.filter(_.Release_Year == 2023)\n",
+    "tbl_2024 = tbl.filter(_.Release_Year == 2024)\n",
+    "# old method\n",
+    "#intersects = tbl_2024.anti_join(tbl_2023, _.SHAPE.intersects(tbl_2023.SHAPE))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 57,
+   "id": "34e264c1-e6d4-4982-b8a7-5b856a3d22c3",
+   "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "64cd02336fac4d9884e2c7ed66472146",
+       "model_id": "11300de9b00446b88e10c2e4f41d81ca",
        "version_major": 2,
        "version_minor": 0
       },
@@ -260,47 +320,180 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "9729a29cd0f345fd9a9bc46dec111744",
+       "model_id": "8ca97fd0f9d74a898d03fa41497c4836",
        "version_major": 2,
-       "version_minor": 1
+       "version_minor": 0
       },
       "text/plain": [
-       "Map(height='600px', map_options={'bearing': 0, 'center': (0, 20), 'pitch': 0, 'style': 'https://basemaps.carto…"
+       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
       ]
      },
-     "execution_count": 162,
      "metadata": {},
-     "output_type": "execute_result"
+     "output_type": "display_data"
+    },
+    {
+     "ename": "InternalException",
+     "evalue": "INTERNAL Error: Calling StringValue::Get on a NULL value\nThis error signals an assertion failure within DuckDB. This usually occurs due to unexpected conditions or errors in the program's logic.\nFor more information, see https://duckdb.org/docs/dev/internal_errors",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mInternalException\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[57], line 9\u001b[0m\n\u001b[1;32m      7\u001b[0m con\u001b[38;5;241m.\u001b[39minstall_extension(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspatial\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m      8\u001b[0m con\u001b[38;5;241m.\u001b[39mload_extension(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspatial\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 9\u001b[0m \u001b[43mdb\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msql\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'''\u001b[39;49m\n\u001b[1;32m     10\u001b[0m \u001b[38;5;124;43mCOPY(\u001b[39;49m\n\u001b[1;32m     11\u001b[0m \u001b[38;5;124;43m  SELECT ST_DIFFERENCE(_2024.geom, _2023.geom) AS geom,\u001b[39;49m\n\u001b[1;32m     12\u001b[0m \u001b[38;5;124;43m  FROM ST_Read(\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mla_2024.geojson\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m) _2024\u001b[39;49m\n\u001b[1;32m     13\u001b[0m \u001b[38;5;124;43m  LEFT JOIN (\u001b[39;49m\n\u001b[1;32m     14\u001b[0m \u001b[38;5;124;43m      SELECT *\u001b[39;49m\n\u001b[1;32m     15\u001b[0m \u001b[38;5;124;43m      FROM ST_Read(\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mla_2023.geojson\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m)\u001b[39;49m\n\u001b[1;32m     16\u001b[0m \u001b[38;5;124;43m  ) _2023\u001b[39;49m\n\u001b[1;32m     17\u001b[0m \u001b[38;5;124;43m  ON ST_INTERSECTS(_2024.geom,_2023.geom)\u001b[39;49m\n\u001b[1;32m     18\u001b[0m \u001b[38;5;124;43m) TO \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43msimple.geojson\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m WITH (FORMAT GDAL, DRIVER \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mGeoJSON\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m, LAYER_CREATION_OPTIONS \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mWRITE_BBOX=YES\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m)\u001b[39;49m\n\u001b[1;32m     19\u001b[0m \u001b[38;5;124;43m'''\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m     21\u001b[0m \u001b[38;5;66;03m#  IF(_2023.objectid IS NULL, _2024.geom, ST_DIFFERENCE(_2024.geom, _2023.geom)) AS geom,\u001b[39;00m\n",
+      "\u001b[0;31mInternalException\u001b[0m: INTERNAL Error: Calling StringValue::Get on a NULL value\nThis error signals an assertion failure within DuckDB. This usually occurs due to unexpected conditions or errors in the program's logic.\nFor more information, see https://duckdb.org/docs/dev/internal_errors"
+     ]
     }
    ],
    "source": [
-    "import leafmap.maplibregl as leafmap\n",
-    "import ibis\n",
-    "from ibis import _\n",
-    "## Minimal look\n",
-    "areas = (\n",
-    "    con.read_parquet(\"https://data.source.coop/cboettig/ca30x30/ca_areas.parquet\")\n",
-    "    .cast({\"SHAPE\": \"geometry\"})\n",
-    "    .rename(geom = \"SHAPE\")\n",
-    "    .select(\"Release_Year\", \"reGAP\", \"OBJECTID\", \"geom\")\n",
-    "    .mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\"))\n",
-    "    \n",
-    ")\n",
+    "import duckdb\n",
     "\n",
-    "#areas.filter(_.Release_Year join(la_2023, _.geom.intersects(la_2023.geom), how=\"left\")\n",
+    "tbl_2023.execute().to_file(\"la_2023.geojson\")\n",
+    "tbl_2024.execute().to_file(\"la_2024.geojson\")\n",
     "\n",
+    "con = duckdb.connect()\n",
+    "con.install_extension(\"spatial\")\n",
+    "con.load_extension(\"spatial\")\n",
+    "db.sql('''\n",
+    "COPY(\n",
+    "  SELECT ST_DIFFERENCE(_2024.geom, _2023.geom) AS geom,\n",
+    "  FROM ST_Read('la_2024.geojson') _2024\n",
+    "  LEFT JOIN (\n",
+    "      SELECT *\n",
+    "      FROM ST_Read('la_2023.geojson')\n",
+    "  ) _2023\n",
+    "  ON ST_INTERSECTS(_2024.geom,_2023.geom)\n",
+    ") TO 'simple.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON', LAYER_CREATION_OPTIONS 'WRITE_BBOX=YES')\n",
+    "''')\n",
     "\n",
-    "new = areas.filter(_.OBJECTID == 111913)\n",
-    "old = areas.filter(_.OBJECTID == 21105)\n",
-    "# old.geom.difference(new.geom)\n",
+    "#  IF(_2023.objectid IS NULL, _2024.geom, ST_DIFFERENCE(_2024.geom, _2023.geom)) AS geom,\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "21dbc7c4-20e2-4422-ac21-cd50badcae58",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import duckdb\n",
+    "db = duckdb.connect()\n",
+    "db.install_extension(\"spatial\")\n",
+    "db.load_extension(\"spatial\")\n",
+    "\n",
+    "\n",
+    "db.sql('''\n",
+    "CREATE OR REPLACE TABLE t1 AS SELECT OBJECTID AS gid, geom, FROM st_read(\"la_2024.geojson\");\n",
+    "CREATE OR REPLACE TABLE t2 AS SELECT OBJECTID AS gid, geom, FROM st_read(\"la_2023.geojson\");\n",
+    "''')\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "d467548c-892f-4786-bfb2-0599f6d3d159",
+   "metadata": {},
+   "outputs": [],
+   "source": [
     "\n",
+    "db.sql('''\n",
+    "COPY (\n",
+    "with temp as \n",
+    "(\n",
+    "  select   b.gid, st_union_agg(a.geom) as geom\n",
+    "  from     t1 b join t2 a on st_intersects(a.geom, b.geom)\n",
+    "  group by b.gid\n",
+    ") \n",
+    "select st_difference(b.geom,coalesce(t.geom, 'GEOMETRYCOLLECTION EMPTY'::geometry)) as geom\n",
+    "from t1 b left join temp t on b.gid = t.gid\n",
+    ") TO 'simple.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON', LAYER_CREATION_OPTIONS 'WRITE_BBOX=YES')\n",
+    "''')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "d869c220-a4c2-46b1-9e02-d5cce5a2bffa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import ibis\n",
+    "from ibis import _\n",
+    "con = ibis.duckdb.connect()\n",
+    "diff = con.read_geo(\"simple.geojson\").mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\"))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "598d096a-5cae-4ddc-938c-908eba334b91",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "9eae57f133a44d0799e4622fdb318e30",
+       "version_major": 2,
+       "version_minor": 1
+      },
+      "text/plain": [
+       "Map(height='600px', map_options={'bearing': 0, 'center': (0, 20), 'pitch': 0, 'style': 'https://basemaps.carto…"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import leafmap.maplibregl as leafmap\n",
     "\n",
     "m = leafmap.Map(style=\"positron\")\n",
-    "#m.add_gdf(old.execute(),layer_type=\"fill\", name = \"2023\", paint=paint)\n",
-    "m.add_gdf(new.execute(),layer_type=\"fill\", name = \"2024\", paint=paint)\n",
+    "#m.add_gdf(la_2023.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"la_2023\", paint = {\"fill-opacity\": 0.5})\n",
+    "#m.add_gdf(la_2024.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"la_2024\", paint = {\"fill-opacity\": 0.5})\n",
+    "m.add_gdf(diff.execute(),layer_type=\"fill\", name = \"diff\", paint = {\"fill-opacity\": 0.5})\n",
     "m.add_layer_control()\n",
-    "m"
+    "m\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d7c08912-3c8a-4810-b046-ea46008880ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import duckdb\n",
+    "\n",
+    "con = duckdb.connect()\n",
+    "con.install_extension(\"spatial\")\n",
+    "con.load_extension(\"spatial\")\n",
+    "con.sql('''\n",
+    "COPY(\n",
+    "WITH geom1 AS (\n",
+    "  SELECT geom\n",
+    "  FROM ST_Read('new.geojson')\n",
+    "),\n",
+    "geom2 AS (\n",
+    "  SELECT geom\n",
+    "  FROM ST_Read('old.geojson')\n",
+    ")\n",
+    "SELECT ST_Difference(\n",
+    "   (SELECT geom FROM geom1),\n",
+    "   (SELECT geom FROM geom2)\n",
+    ") AS geom\n",
+    ") TO 'simple.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON', LAYER_CREATION_OPTIONS 'WRITE_BBOX=YES')\n",
+    "''')\n",
+    "\n"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b47fd840-a918-4690-adc0-ca074d09932c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {