maybe got it

explore.ipynb

@@ -2,7 +2,7 @@
  "cells": [
  {
  "cell_type": "code",
- "execution_count": 6,
+ "execution_count": 16,
  "id": "f7e6298c-d886-432a-a1b7-c3fee914c24f",
  "metadata": {},
  "outputs": [],
@@ -20,26 +20,10 @@
  },
  {
  "cell_type": "code",
- "execution_count": 104,
+ "execution_count": null,
  "id": "bd9c2561-334a-40d6-adf0-ce20a991a9a7",
  "metadata": {},
- "outputs": [
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "f46fb7de69784a4e89763c5432d06a40",
- "version_major": 2,
- "version_minor": 1
- },
- "text/plain": [
- "Map(height='600px', map_options={'bearing': 0, 'center': (0, 20), 'pitch': 0, 'style': 'https://basemaps.carto…"
- ]
- },
- "execution_count": 104,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
  "source": [
  "# Visualize the full data to show overlap:\n",
  "import leafmap.maplibregl as leafmap\n",
@@ -76,7 +60,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 158,
+ "execution_count": 10,
  "id": "3ceed4c7-b7a2-436e-bca1-c57e7133cdf6",
  "metadata": {},
  "outputs": [],
@@ -84,9 +68,8 @@
  "# Make things faster. Let's zoom in on a subset of the data. \n",
  "\n",
  "from ibis.interactive import *\n",
- "\n",
  "ca_parquet = \"/home/rstudio/source.coop/cboettig/ca30x30/ca_areas.parquet\"\n",
- "con = ibis.duckdb.connect()\n",
+ "conn = ibis.duckdb.connect()\n",
  "\n",
  "# Ugly code for a cookie cutter zoom\n",
  "# Turns out we could have just used filter(_.UNIT_NAME == \"Angeles National Forest\")\n",
@@ -98,19 +81,41 @@
  "cookie_cutter = ibis.literal(gdf.geometry.iloc[0])\n",
  "\n",
  "angeles_forest = (\n",
- " con.read_parquet(ca_parquet)\n",
+ " conn.read_parquet(ca_parquet)\n",
  " .cast({\"SHAPE\": \"geometry\"})\n",
- " .mutate(geom = _.SHAPE.convert(\"epsg:3310\",\"epsg:4326\"))\n",
- " .filter( _.geom.intersects(cookie_cutter))\n",
+ " .rename(geom = \"SHAPE\")\n",
+ " # .mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\"))\n",
+ " # .filter( _.geom.intersects(cookie_cutter))\n",
  " .filter(_.UNIT_NAME == \"Angeles National Forest\") \n",
  " .select(\"reGAP\", \"UNIT_NAME\", \"MNG_AGNCY\", \"Release_Year\", \"OBJECTID\", \"geom\")\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a0cb34b1-8d70-49bf-80c6-244ecc8ddf84",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import ibis\n",
+ "from ibis import _\n",
+ "\n",
+ "tbl = (\n",
+ " conn.read_parquet(\"https://data.source.coop/cboettig/ca30x30/ca_areas.parquet\")\n",
+ " .cast({\"SHAPE\": \"geometry\"})\n",
+ " .filter(_.UNIT_NAME == \"Angeles National Forest\", _.reGAP < 3) \n",
  ")\n",
- "\n"
+ "\n",
+ "tbl_2023 = tbl.filter(_.Release_Year == 2023)\n",
+ "tbl_2024 = tbl.filter(_.Release_Year == 2024)\n",
+ "intersects = tbl_2024.anti_join(tbl_2023, _.SHAPE.intersects(tbl_2023.SHAPE))\n",
+ "intersects"
  ]
  },
  {
  "cell_type": "code",
- "execution_count": 159,
+ "execution_count": 11,
  "id": "3ed21429-a75e-4b52-b6eb-d41c8f051883",
  "metadata": {},
  "outputs": [],
@@ -122,7 +127,7 @@
  },
  {
  "cell_type": "code",
- "execution_count": 160,
+ "execution_count": 12,
  "id": "7bfac88a-822a-4e4e-883e-b7e51544abd1",
  "metadata": {},
  "outputs": [],
@@ -133,21 +138,23 @@
  "\n",
  "# THIS IS ALMOST WHAT WE WANT, BUT for some issues\n",
  "# All reGAP 1,2 polygons in 2024 that don't intersect 2023\n",
- "la_intersects = la_2024.anti_join(la_2023, _.geom.intersects(la_2023.geom))\n"
+ "la_intersects = la_2024.anti_join(la_2023, _.geom.intersects(la_2023.geom))\n",
+ "\n",
+ "#la_contains = la_2024.anti_join(la_2023, _.geom.contains(la_2023.geom.centroid()))\n",
+ "#centroids = la_2023.filter(_.reGAP == 2).mutate( geom = la_2023.filter(_.reGAP == 2).geom.centroid() )\n",
+ "#centroids = old.mutate( geom = old.geom.centroid() )"
  ]
  },
  {
  "cell_type": "code",
- "execution_count": 164,
+ "execution_count": 23,
  "id": "ddae1f9e-988a-4aed-bc71-f1c40d2bb0ea",
- "metadata": {
- "scrolled": true
- },
+ "metadata": {},
  "outputs": [
  {
  "data": {
  "application/vnd.jupyter.widget-view+json": {
- "model_id": "52b3b9c8cb1b4bbf8fe16aebef4d2e25",
+ "model_id": "ff9365d5ed2848399f1a00952cf6508f",
  "version_major": 2,
  "version_minor": 1
  },
@@ -155,7 +162,7 @@
  "Map(height='600px', map_options={'bearing': 0, 'center': (0, 20), 'pitch': 0, 'style': 'https://basemaps.carto…"
  ]
  },
- "execution_count": 164,
+ "execution_count": 23,
  "metadata": {},
  "output_type": "execute_result"
  }
@@ -179,51 +186,21 @@
  "paint = {\"fill-color\": gap, \"fill-opacity\": 0.5}\n",
  "\n",
  "m = leafmap.Map(style=\"positron\")\n",
- "m.add_gdf(allgap_2023.execute(),layer_type=\"fill\", name = \"allgap_2023\", paint=paint)\n",
- "m.add_gdf(allgap_2024.execute(),layer_type=\"fill\", name = \"allgap_2024\", paint=paint)\n",
- "#m.add_gdf(la_2023.execute(),layer_type=\"fill\", name = \"2023\", paint=paint)\n",
- "#m.add_gdf(la_2024.execute(),layer_type=\"fill\", name = \"2024\", paint=paint)\n",
- "m.add_gdf(la_intersects.execute(),layer_type=\"fill\", name = \"anti_intersects\", paint=paint)\n",
- "#m.add_gdf(intersects.execute(), layer_type=\"fill\", name = \"overlapping\", paint=paint)\n",
+ "#m.add_gdf(allgap_2023.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"allgap_2023\", paint=paint)\n",
+ "#m.add_gdf(allgap_2024.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"allgap_2024\", paint=paint)\n",
+ "m.add_gdf(la_2023.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"2023\", paint=paint)\n",
+ "#m.add_gdf(la_2024.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"2024\", paint=paint)\n",
+ "#m.add_gdf(la_intersects.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"anti_intersects\", paint=paint)\n",
  "m.add_layer_control()\n",
  "m"
  ]
  },
  {
  "cell_type": "code",
- "execution_count": 170,
+ "execution_count": null,
  "id": "6832038f-0342-495f-8cbf-9a8f3cc83aac",
  "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┓\n",
- "┃<span style=\"font-weight: bold\"> OBJECTID_right </span>┃<span style=\"font-weight: bold\"> geom </span>┃<span style=\"font-weight: bold\"> reGAP </span>┃\n",
- "┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━┩\n",
- "│ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int64</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">geospatial:geometry</span> │ <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">int16</span> │\n",
- "├────────────────┼──────────────────────────────────────────────────────────────────────────────────┼───────┤\n",
- "│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">21105</span> │ <span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">MULTIPOLYGON</span><span style=\"color: #000000; text-decoration-color: #000000\"> Z </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">(((</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-118.017</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34.499</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-118.019</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34.499</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-118.019</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34.499</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #808000; text-decoration-color: #808000\">...</span><span style=\"font-weight: bold\">&gt;</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> │\n",
- "│ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">22618</span> │ <span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">MULTIPOLYGON</span><span style=\"color: #000000; text-decoration-color: #000000\"> Z </span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">(((</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-118.297</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34.433</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-118.298</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34.433</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-118.303</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34.433</span><span style=\"color: #000000; text-decoration-color: #000000\"> </span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"color: #000000; text-decoration-color: #000000\">, </span><span style=\"color: #808000; text-decoration-color: #808000\">...</span><span style=\"font-weight: bold\">&gt;</span> │ <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> │\n",
- "└────────────────┴──────────────────────────────────────────────────────────────────────────────────┴───────┘\n",
- "</pre>\n"
- ],
- "text/plain": [
- "┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┓\n",
- "┃\u001b[1m \u001b[0m\u001b[1mOBJECTID_right\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mgeom\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mreGAP\u001b[0m\u001b[1m \u001b[0m┃\n",
- "┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━┩\n",
- "│ \u001b[2mint64\u001b[0m │ \u001b[2mgeospatial:geometry\u001b[0m │ \u001b[2mint16\u001b[0m │\n",
- "├────────────────┼──────────────────────────────────────────────────────────────────────────────────┼───────┤\n",
- "│ \u001b[1;36m21105\u001b[0m │ \u001b[1m<\u001b[0m\u001b[1;95mMULTIPOLYGON\u001b[0m\u001b[39m Z \u001b[0m\u001b[1;39m(\u001b[0m\u001b[1;39m(\u001b[0m\u001b[1;39m(\u001b[0m\u001b[1;36m-118.017\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m34.499\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m0\u001b[0m\u001b[39m, \u001b[0m\u001b[1;36m-118.019\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m34.499\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m0\u001b[0m\u001b[39m, \u001b[0m\u001b[1;36m-118.019\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m34.499\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m0\u001b[0m\u001b[39m, \u001b[0m\u001b[33m...\u001b[0m\u001b[1m>\u001b[0m │ \u001b[1;36m2\u001b[0m │\n",
- "�� \u001b[1;36m22618\u001b[0m │ \u001b[1m<\u001b[0m\u001b[1;95mMULTIPOLYGON\u001b[0m\u001b[39m Z \u001b[0m\u001b[1;39m(\u001b[0m\u001b[1;39m(\u001b[0m\u001b[1;39m(\u001b[0m\u001b[1;36m-118.297\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m34.433\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m0\u001b[0m\u001b[39m, \u001b[0m\u001b[1;36m-118.298\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m34.433\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m0\u001b[0m\u001b[39m, \u001b[0m\u001b[1;36m-118.303\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m34.433\u001b[0m\u001b[39m \u001b[0m\u001b[1;36m0\u001b[0m\u001b[39m, \u001b[0m\u001b[33m...\u001b[0m\u001b[1m>\u001b[0m │ \u001b[1;36m2\u001b[0m │\n",
- "└────────────────┴──────────────────────────────────────────────────────────────────────────────────┴───────┘"
- ]
- },
- "execution_count": 170,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
  "source": [
  "## It intersects these, even though it looks like it shouldn't\n",
  "\n",
@@ -239,14 +216,97 @@
  },
  {
  "cell_type": "code",
- "execution_count": 162,
+ "execution_count": null,
  "id": "b41b0af6-635c-4ffb-9f19-342284f18e62",
  "metadata": {},
+ "outputs": [],
+ "source": [
+ "import leafmap.maplibregl as leafmap\n",
+ "import ibis\n",
+ "from ibis import _\n",
+ "## Minimal look\n",
+ "areas = (\n",
+ " conn.read_parquet(\"https://data.source.coop/cboettig/ca30x30/ca_areas.parquet\")\n",
+ " .cast({\"SHAPE\": \"geometry\"})\n",
+ " .rename(geom = \"SHAPE\")\n",
+ " .select(\"Release_Year\", \"reGAP\", \"OBJECTID\", \"geom\")\n",
+ " .mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\"))\n",
+ " \n",
+ ")\n",
+ "new = areas.filter(_.OBJECTID == 111913)\n",
+ "old = areas.filter(_.OBJECTID == 21105)\n",
+ "# old.geom.difference(new.geom)\n",
+ "\n",
+ "\n",
+ "m = leafmap.Map(style=\"positron\")\n",
+ "m.add_gdf(old.execute(),layer_type=\"fill\", name = \"2023\", paint=paint)\n",
+ "m.add_gdf(new.execute(),layer_type=\"fill\", name = \"2024\", paint=paint)\n",
+ "m.add_layer_control()\n",
+ "m"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ab985b4f-2257-46ce-95af-508c17323ca3",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "id": "96cefbe6-12c9-4ca4-9352-c3573924c91c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "la_2023.execute().to_file(\"la_2023.geojson\")\n",
+ "la_2024.execute().to_file(\"la_2024.geojson\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4b9ee360-8583-4096-b1e1-43772f32a182",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "la_2024.filter(_.OBJECTID == 111913).execute().to_file(\"new.geojson\")\n",
+ "la_2023.filter(_.OBJECTID == 21105).execute().to_file(\"old.geojson\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 41,
+ "id": "1e538620-19f5-42eb-8233-e280b5c5b920",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import ibis\n",
+ "from ibis import _\n",
+ "\n",
+ "tbl = (\n",
+ " conn.read_parquet(\"https://data.source.coop/cboettig/ca30x30/ca_areas.parquet\")\n",
+ " .cast({\"SHAPE\": \"geometry\"})\n",
+ " .filter(_.UNIT_NAME == \"Angeles National Forest\", _.reGAP < 3) \n",
+ ")\n",
+ "\n",
+ "tbl_2023 = tbl.filter(_.Release_Year == 2023)\n",
+ "tbl_2024 = tbl.filter(_.Release_Year == 2024)\n",
+ "# old method\n",
+ "#intersects = tbl_2024.anti_join(tbl_2023, _.SHAPE.intersects(tbl_2023.SHAPE))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 57,
+ "id": "34e264c1-e6d4-4982-b8a7-5b856a3d22c3",
+ "metadata": {},
  "outputs": [
  {
  "data": {
  "application/vnd.jupyter.widget-view+json": {
- "model_id": "64cd02336fac4d9884e2c7ed66472146",
+ "model_id": "11300de9b00446b88e10c2e4f41d81ca",
  "version_major": 2,
  "version_minor": 0
  },
@@ -260,47 +320,180 @@
  {
  "data": {
  "application/vnd.jupyter.widget-view+json": {
- "model_id": "9729a29cd0f345fd9a9bc46dec111744",
+ "model_id": "8ca97fd0f9d74a898d03fa41497c4836",
  "version_major": 2,
- "version_minor": 1
+ "version_minor": 0
  },
  "text/plain": [
- "Map(height='600px', map_options={'bearing': 0, 'center': (0, 20), 'pitch': 0, 'style': 'https://basemaps.carto…"
+ "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
  ]
  },
- "execution_count": 162,
  "metadata": {},
- "output_type": "execute_result"
+ "output_type": "display_data"
+ },
+ {
+ "ename": "InternalException",
+ "evalue": "INTERNAL Error: Calling StringValue::Get on a NULL value\nThis error signals an assertion failure within DuckDB. This usually occurs due to unexpected conditions or errors in the program's logic.\nFor more information, see https://duckdb.org/docs/dev/internal_errors",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mInternalException\u001b[0m Traceback (most recent call last)",
+ "Cell \u001b[0;32mIn[57], line 9\u001b[0m\n\u001b[1;32m 7\u001b[0m con\u001b[38;5;241m.\u001b[39minstall_extension(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspatial\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 8\u001b[0m con\u001b[38;5;241m.\u001b[39mload_extension(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspatial\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 9\u001b[0m \u001b[43mdb\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msql\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'''\u001b[39;49m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;124;43mCOPY(\u001b[39;49m\n\u001b[1;32m 11\u001b[0m \u001b[38;5;124;43m SELECT ST_DIFFERENCE(_2024.geom, _2023.geom) AS geom,\u001b[39;49m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;124;43m FROM ST_Read(\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mla_2024.geojson\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m) _2024\u001b[39;49m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;124;43m LEFT JOIN (\u001b[39;49m\n\u001b[1;32m 14\u001b[0m \u001b[38;5;124;43m SELECT *\u001b[39;49m\n\u001b[1;32m 15\u001b[0m \u001b[38;5;124;43m FROM ST_Read(\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mla_2023.geojson\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m)\u001b[39;49m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;124;43m ) _2023\u001b[39;49m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;124;43m ON ST_INTERSECTS(_2024.geom,_2023.geom)\u001b[39;49m\n\u001b[1;32m 18\u001b[0m \u001b[38;5;124;43m) TO \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43msimple.geojson\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m WITH (FORMAT GDAL, DRIVER \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mGeoJSON\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m, LAYER_CREATION_OPTIONS \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mWRITE_BBOX=YES\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m)\u001b[39;49m\n\u001b[1;32m 19\u001b[0m \u001b[38;5;124;43m'''\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 21\u001b[0m \u001b[38;5;66;03m# IF(_2023.objectid IS NULL, _2024.geom, ST_DIFFERENCE(_2024.geom, _2023.geom)) AS geom,\u001b[39;00m\n",
+ "\u001b[0;31mInternalException\u001b[0m: INTERNAL Error: Calling StringValue::Get on a NULL value\nThis error signals an assertion failure within DuckDB. This usually occurs due to unexpected conditions or errors in the program's logic.\nFor more information, see https://duckdb.org/docs/dev/internal_errors"
+ ]
  }
  ],
  "source": [
- "import leafmap.maplibregl as leafmap\n",
- "import ibis\n",
- "from ibis import _\n",
- "## Minimal look\n",
- "areas = (\n",
- " con.read_parquet(\"https://data.source.coop/cboettig/ca30x30/ca_areas.parquet\")\n",
- " .cast({\"SHAPE\": \"geometry\"})\n",
- " .rename(geom = \"SHAPE\")\n",
- " .select(\"Release_Year\", \"reGAP\", \"OBJECTID\", \"geom\")\n",
- " .mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\"))\n",
- " \n",
- ")\n",
+ "import duckdb\n",
  "\n",
- "#areas.filter(_.Release_Year join(la_2023, _.geom.intersects(la_2023.geom), how=\"left\")\n",
+ "tbl_2023.execute().to_file(\"la_2023.geojson\")\n",
+ "tbl_2024.execute().to_file(\"la_2024.geojson\")\n",
  "\n",
+ "con = duckdb.connect()\n",
+ "con.install_extension(\"spatial\")\n",
+ "con.load_extension(\"spatial\")\n",
+ "db.sql('''\n",
+ "COPY(\n",
+ " SELECT ST_DIFFERENCE(_2024.geom, _2023.geom) AS geom,\n",
+ " FROM ST_Read('la_2024.geojson') _2024\n",
+ " LEFT JOIN (\n",
+ " SELECT *\n",
+ " FROM ST_Read('la_2023.geojson')\n",
+ " ) _2023\n",
+ " ON ST_INTERSECTS(_2024.geom,_2023.geom)\n",
+ ") TO 'simple.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON', LAYER_CREATION_OPTIONS 'WRITE_BBOX=YES')\n",
+ "''')\n",
  "\n",
- "new = areas.filter(_.OBJECTID == 111913)\n",
- "old = areas.filter(_.OBJECTID == 21105)\n",
- "# old.geom.difference(new.geom)\n",
+ "# IF(_2023.objectid IS NULL, _2024.geom, ST_DIFFERENCE(_2024.geom, _2023.geom)) AS geom,\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "21dbc7c4-20e2-4422-ac21-cd50badcae58",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import duckdb\n",
+ "db = duckdb.connect()\n",
+ "db.install_extension(\"spatial\")\n",
+ "db.load_extension(\"spatial\")\n",
+ "\n",
+ "\n",
+ "db.sql('''\n",
+ "CREATE OR REPLACE TABLE t1 AS SELECT OBJECTID AS gid, geom, FROM st_read(\"la_2024.geojson\");\n",
+ "CREATE OR REPLACE TABLE t2 AS SELECT OBJECTID AS gid, geom, FROM st_read(\"la_2023.geojson\");\n",
+ "''')\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "d467548c-892f-4786-bfb2-0599f6d3d159",
+ "metadata": {},
+ "outputs": [],
+ "source": [
  "\n",
+ "db.sql('''\n",
+ "COPY (\n",
+ "with temp as \n",
+ "(\n",
+ " select b.gid, st_union_agg(a.geom) as geom\n",
+ " from t1 b join t2 a on st_intersects(a.geom, b.geom)\n",
+ " group by b.gid\n",
+ ") \n",
+ "select st_difference(b.geom,coalesce(t.geom, 'GEOMETRYCOLLECTION EMPTY'::geometry)) as geom\n",
+ "from t1 b left join temp t on b.gid = t.gid\n",
+ ") TO 'simple.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON', LAYER_CREATION_OPTIONS 'WRITE_BBOX=YES')\n",
+ "''')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "id": "d869c220-a4c2-46b1-9e02-d5cce5a2bffa",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import ibis\n",
+ "from ibis import _\n",
+ "con = ibis.duckdb.connect()\n",
+ "diff = con.read_geo(\"simple.geojson\").mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\"))\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "id": "598d096a-5cae-4ddc-938c-908eba334b91",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "9eae57f133a44d0799e4622fdb318e30",
+ "version_major": 2,
+ "version_minor": 1
+ },
+ "text/plain": [
+ "Map(height='600px', map_options={'bearing': 0, 'center': (0, 20), 'pitch': 0, 'style': 'https://basemaps.carto…"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import leafmap.maplibregl as leafmap\n",
  "\n",
  "m = leafmap.Map(style=\"positron\")\n",
- "#m.add_gdf(old.execute(),layer_type=\"fill\", name = \"2023\", paint=paint)\n",
- "m.add_gdf(new.execute(),layer_type=\"fill\", name = \"2024\", paint=paint)\n",
+ "#m.add_gdf(la_2023.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"la_2023\", paint = {\"fill-opacity\": 0.5})\n",
+ "#m.add_gdf(la_2024.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"la_2024\", paint = {\"fill-opacity\": 0.5})\n",
+ "m.add_gdf(diff.execute(),layer_type=\"fill\", name = \"diff\", paint = {\"fill-opacity\": 0.5})\n",
  "m.add_layer_control()\n",
- "m"
+ "m\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d7c08912-3c8a-4810-b046-ea46008880ca",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import duckdb\n",
+ "\n",
+ "con = duckdb.connect()\n",
+ "con.install_extension(\"spatial\")\n",
+ "con.load_extension(\"spatial\")\n",
+ "con.sql('''\n",
+ "COPY(\n",
+ "WITH geom1 AS (\n",
+ " SELECT geom\n",
+ " FROM ST_Read('new.geojson')\n",
+ "),\n",
+ "geom2 AS (\n",
+ " SELECT geom\n",
+ " FROM ST_Read('old.geojson')\n",
+ ")\n",
+ "SELECT ST_Difference(\n",
+ " (SELECT geom FROM geom1),\n",
+ " (SELECT geom FROM geom2)\n",
+ ") AS geom\n",
+ ") TO 'simple.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON', LAYER_CREATION_OPTIONS 'WRITE_BBOX=YES')\n",
+ "''')\n",
+ "\n"
  ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b47fd840-a918-4690-adc0-ca074d09932c",
+ "metadata": {},
+ "outputs": [],
+ "source": []
  }
  ],
  "metadata": {