two solns

explore.ipynb

@@ -60,7 +60,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 17,
    "id": "3ceed4c7-b7a2-436e-bca1-c57e7133cdf6",
    "metadata": {},
    "outputs": [],
@@ -86,91 +86,61 @@
     "    .rename(geom = \"SHAPE\")\n",
     " #   .mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\"))\n",
     " #   .filter( _.geom.intersects(cookie_cutter))\n",
-    "    .filter(_.UNIT_NAME == \"Angeles National Forest\") \n",
+    " #   .filter(_.UNIT_NAME == \"Angeles National Forest\") \n",
     "    .select(\"reGAP\", \"UNIT_NAME\", \"MNG_AGNCY\", \"Release_Year\", \"OBJECTID\", \"geom\")\n",
-    ")"
+    ")\n",
+    "# split 2023 & 2024\n",
+    "allgap_2023 = angeles_forest.filter(_.Release_Year == 2023)\n",
+    "allgap_2024 = angeles_forest.filter(_.Release_Year == 2024)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 25,
    "id": "a0cb34b1-8d70-49bf-80c6-244ecc8ddf84",
    "metadata": {},
    "outputs": [],
    "source": [
     "import ibis\n",
     "from ibis import _\n",
+    "buffer = -0.0003 \n",
     "\n",
     "tbl = (\n",
     "    conn.read_parquet(\"https://data.source.coop/cboettig/ca30x30/ca_areas.parquet\")\n",
     "    .cast({\"SHAPE\": \"geometry\"})\n",
-    "    .filter(_.UNIT_NAME == \"Angeles National Forest\", _.reGAP < 3) \n",
+    "    .rename(geom = \"SHAPE\")\n",
+    "  #  .filter(_.UNIT_NAME == \"Angeles National Forest\")\n",
+    "    .filter(_.reGAP < 3) \n",
+    "    .mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\"))\n",
     ")\n",
-    "\n",
-    "tbl_2023 = tbl.filter(_.Release_Year == 2023)\n",
+    "tbl_2023 = tbl.filter(_.Release_Year == 2023).mutate(geom=_.geom.buffer(buffer))\n",
     "tbl_2024 = tbl.filter(_.Release_Year == 2024)\n",
-    "intersects = tbl_2024.anti_join(tbl_2023, _.SHAPE.intersects(tbl_2023.SHAPE))\n",
-    "intersects"
+    "intersects = tbl_2024.anti_join(tbl_2023, _.geom.intersects(tbl_2023.geom))"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "id": "3ed21429-a75e-4b52-b6eb-d41c8f051883",
    "metadata": {},
-   "outputs": [],
-   "source": [
-    "# split 2023 & 2024\n",
-    "allgap_2023 = angeles_forest.filter(_.Release_Year == 2023)\n",
-    "allgap_2024 = angeles_forest.filter(_.Release_Year == 2024)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "id": "7bfac88a-822a-4e4e-883e-b7e51544abd1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# split but with reGAP < 3\n",
-    "la_2023 = allgap_2023.filter(_.reGAP < 3)\n",
-    "la_2024 = allgap_2024.filter(_.reGAP < 3)\n",
-    "\n",
-    "# THIS IS ALMOST WHAT WE WANT, BUT for some issues\n",
-    "# All reGAP 1,2 polygons in 2024 that don't intersect 2023\n",
-    "la_intersects = la_2024.anti_join(la_2023, _.geom.intersects(la_2023.geom))\n",
-    "\n",
-    "#la_contains = la_2024.anti_join(la_2023, _.geom.contains(la_2023.geom.centroid()))\n",
-    "#centroids = la_2023.filter(_.reGAP == 2).mutate( geom = la_2023.filter(_.reGAP == 2).geom.centroid() )\n",
-    "#centroids = old.mutate( geom = old.geom.centroid() )"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 23,
-   "id": "ddae1f9e-988a-4aed-bc71-f1c40d2bb0ea",
-   "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ff9365d5ed2848399f1a00952cf6508f",
+       "model_id": "7c5bd159278847f481827b444ffc32b8",
        "version_major": 2,
-       "version_minor": 1
+       "version_minor": 0
       },
       "text/plain": [
-       "Map(height='600px', map_options={'bearing': 0, 'center': (0, 20), 'pitch': 0, 'style': 'https://basemaps.carto…"
+       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
       ]
      },
-     "execution_count": 23,
      "metadata": {},
-     "output_type": "execute_result"
+     "output_type": "display_data"
     }
    ],
    "source": [
     "import leafmap.maplibregl as leafmap\n",
-    "#la_gdf = la_2024.execute()\n",
-    "\n",
     "\n",
     "release_year = {'property': 'Release_Year', 'type': 'categorical','stops': [[2023, \"#FF000080\"], [2024, \"#0000FF80\"]]},\n",
     "gap = {\n",
@@ -182,97 +152,20 @@
     "            [3, \"#BBBBBB\"],\n",
     "            [4, \"#F8F8F8\"]]\n",
     "        }\n",
-    "\n",
     "paint = {\"fill-color\": gap, \"fill-opacity\": 0.5}\n",
     "\n",
     "m = leafmap.Map(style=\"positron\")\n",
-    "#m.add_gdf(allgap_2023.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"allgap_2023\", paint=paint)\n",
-    "#m.add_gdf(allgap_2024.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"allgap_2024\", paint=paint)\n",
-    "m.add_gdf(la_2023.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"2023\", paint=paint)\n",
-    "#m.add_gdf(la_2024.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"2024\", paint=paint)\n",
-    "#m.add_gdf(la_intersects.mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\")).execute(),layer_type=\"fill\", name = \"anti_intersects\", paint=paint)\n",
+    "m.add_gdf(intersects.execute(),layer_type=\"fill\", name = \"intes\", paint=paint)\n",
     "m.add_layer_control()\n",
     "m"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6832038f-0342-495f-8cbf-9a8f3cc83aac",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "## It intersects these, even though it looks like it shouldn't\n",
-    "\n",
-    "hits = (la_2024\n",
-    " .filter(_.OBJECTID == 111913) # appears to be the new stuff\n",
-    " .join(la_2023, _.geom.intersects(la_2023.geom), how=\"left\")\n",
-    " .select(_.OBJECTID_right, _.geom_right, _.reGAP)\n",
-    " .rename(geom = \"geom_right\")\n",
-    ")\n",
-    "hits\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "b41b0af6-635c-4ffb-9f19-342284f18e62",
+   "cell_type": "markdown",
+   "id": "ebbb2650-4442-4e54-8467-5e681d6fab9e",
    "metadata": {},
-   "outputs": [],
-   "source": [
-    "import leafmap.maplibregl as leafmap\n",
-    "import ibis\n",
-    "from ibis import _\n",
-    "## Minimal look\n",
-    "areas = (\n",
-    "    conn.read_parquet(\"https://data.source.coop/cboettig/ca30x30/ca_areas.parquet\")\n",
-    "    .cast({\"SHAPE\": \"geometry\"})\n",
-    "    .rename(geom = \"SHAPE\")\n",
-    "    .select(\"Release_Year\", \"reGAP\", \"OBJECTID\", \"geom\")\n",
-    "    .mutate(geom = _.geom.convert(\"epsg:3310\",\"epsg:4326\"))\n",
-    "    \n",
-    ")\n",
-    "new = areas.filter(_.OBJECTID == 111913)\n",
-    "old = areas.filter(_.OBJECTID == 21105)\n",
-    "# old.geom.difference(new.geom)\n",
-    "\n",
-    "\n",
-    "m = leafmap.Map(style=\"positron\")\n",
-    "m.add_gdf(old.execute(),layer_type=\"fill\", name = \"2023\", paint=paint)\n",
-    "m.add_gdf(new.execute(),layer_type=\"fill\", name = \"2024\", paint=paint)\n",
-    "m.add_layer_control()\n",
-    "m"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ab985b4f-2257-46ce-95af-508c17323ca3",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "id": "96cefbe6-12c9-4ca4-9352-c3573924c91c",
-   "metadata": {},
-   "outputs": [],
    "source": [
-    "la_2023.execute().to_file(\"la_2023.geojson\")\n",
-    "la_2024.execute().to_file(\"la_2024.geojson\")\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "4b9ee360-8583-4096-b1e1-43772f32a182",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "la_2024.filter(_.OBJECTID == 111913).execute().to_file(\"new.geojson\")\n",
-    "la_2023.filter(_.OBJECTID == 21105).execute().to_file(\"old.geojson\")\n"
+    "Using difference (overlap) instead:"
    ]
   },
   {
@@ -284,7 +177,6 @@
    "source": [
     "import ibis\n",
     "from ibis import _\n",
-    "\n",
     "tbl = (\n",
     "    conn.read_parquet(\"https://data.source.coop/cboettig/ca30x30/ca_areas.parquet\")\n",
     "    .cast({\"SHAPE\": \"geometry\"})\n",
@@ -293,79 +185,9 @@
     "\n",
     "tbl_2023 = tbl.filter(_.Release_Year == 2023)\n",
     "tbl_2024 = tbl.filter(_.Release_Year == 2024)\n",
-    "# old method\n",
-    "#intersects = tbl_2024.anti_join(tbl_2023, _.SHAPE.intersects(tbl_2023.SHAPE))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 57,
-   "id": "34e264c1-e6d4-4982-b8a7-5b856a3d22c3",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "11300de9b00446b88e10c2e4f41d81ca",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "8ca97fd0f9d74a898d03fa41497c4836",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "FloatProgress(value=0.0, layout=Layout(width='auto'), style=ProgressStyle(bar_color='black'))"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "ename": "InternalException",
-     "evalue": "INTERNAL Error: Calling StringValue::Get on a NULL value\nThis error signals an assertion failure within DuckDB. This usually occurs due to unexpected conditions or errors in the program's logic.\nFor more information, see https://duckdb.org/docs/dev/internal_errors",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mInternalException\u001b[0m                         Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[57], line 9\u001b[0m\n\u001b[1;32m      7\u001b[0m con\u001b[38;5;241m.\u001b[39minstall_extension(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspatial\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m      8\u001b[0m con\u001b[38;5;241m.\u001b[39mload_extension(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mspatial\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 9\u001b[0m \u001b[43mdb\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msql\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'''\u001b[39;49m\n\u001b[1;32m     10\u001b[0m \u001b[38;5;124;43mCOPY(\u001b[39;49m\n\u001b[1;32m     11\u001b[0m \u001b[38;5;124;43m  SELECT ST_DIFFERENCE(_2024.geom, _2023.geom) AS geom,\u001b[39;49m\n\u001b[1;32m     12\u001b[0m \u001b[38;5;124;43m  FROM ST_Read(\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mla_2024.geojson\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m) _2024\u001b[39;49m\n\u001b[1;32m     13\u001b[0m \u001b[38;5;124;43m  LEFT JOIN (\u001b[39;49m\n\u001b[1;32m     14\u001b[0m \u001b[38;5;124;43m      SELECT *\u001b[39;49m\n\u001b[1;32m     15\u001b[0m \u001b[38;5;124;43m      FROM ST_Read(\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mla_2023.geojson\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m)\u001b[39;49m\n\u001b[1;32m     16\u001b[0m \u001b[38;5;124;43m  ) _2023\u001b[39;49m\n\u001b[1;32m     17\u001b[0m \u001b[38;5;124;43m  ON ST_INTERSECTS(_2024.geom,_2023.geom)\u001b[39;49m\n\u001b[1;32m     18\u001b[0m \u001b[38;5;124;43m) TO \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43msimple.geojson\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m WITH (FORMAT GDAL, DRIVER \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mGeoJSON\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m, LAYER_CREATION_OPTIONS \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mWRITE_BBOX=YES\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m)\u001b[39;49m\n\u001b[1;32m     19\u001b[0m \u001b[38;5;124;43m'''\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m     21\u001b[0m \u001b[38;5;66;03m#  IF(_2023.objectid IS NULL, _2024.geom, ST_DIFFERENCE(_2024.geom, _2023.geom)) AS geom,\u001b[39;00m\n",
-      "\u001b[0;31mInternalException\u001b[0m: INTERNAL Error: Calling StringValue::Get on a NULL value\nThis error signals an assertion failure within DuckDB. This usually occurs due to unexpected conditions or errors in the program's logic.\nFor more information, see https://duckdb.org/docs/dev/internal_errors"
-     ]
-    }
-   ],
-   "source": [
-    "import duckdb\n",
     "\n",
     "tbl_2023.execute().to_file(\"la_2023.geojson\")\n",
-    "tbl_2024.execute().to_file(\"la_2024.geojson\")\n",
-    "\n",
-    "con = duckdb.connect()\n",
-    "con.install_extension(\"spatial\")\n",
-    "con.load_extension(\"spatial\")\n",
-    "db.sql('''\n",
-    "COPY(\n",
-    "  SELECT ST_DIFFERENCE(_2024.geom, _2023.geom) AS geom,\n",
-    "  FROM ST_Read('la_2024.geojson') _2024\n",
-    "  LEFT JOIN (\n",
-    "      SELECT *\n",
-    "      FROM ST_Read('la_2023.geojson')\n",
-    "  ) _2023\n",
-    "  ON ST_INTERSECTS(_2024.geom,_2023.geom)\n",
-    ") TO 'simple.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON', LAYER_CREATION_OPTIONS 'WRITE_BBOX=YES')\n",
-    "''')\n",
-    "\n",
-    "#  IF(_2023.objectid IS NULL, _2024.geom, ST_DIFFERENCE(_2024.geom, _2023.geom)) AS geom,\n",
-    "\n"
+    "tbl_2024.execute().to_file(\"la_2024.geojson\")\n"
    ]
   },
   {
@@ -380,22 +202,10 @@
     "db.install_extension(\"spatial\")\n",
     "db.load_extension(\"spatial\")\n",
     "\n",
-    "\n",
     "db.sql('''\n",
     "CREATE OR REPLACE TABLE t1 AS SELECT OBJECTID AS gid, geom, FROM st_read(\"la_2024.geojson\");\n",
     "CREATE OR REPLACE TABLE t2 AS SELECT OBJECTID AS gid, geom, FROM st_read(\"la_2023.geojson\");\n",
     "''')\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "id": "d467548c-892f-4786-bfb2-0599f6d3d159",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "\n",
     "db.sql('''\n",
     "COPY (\n",
     "with temp as \n",
@@ -455,45 +265,6 @@
     "m.add_layer_control()\n",
     "m\n"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d7c08912-3c8a-4810-b046-ea46008880ca",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import duckdb\n",
-    "\n",
-    "con = duckdb.connect()\n",
-    "con.install_extension(\"spatial\")\n",
-    "con.load_extension(\"spatial\")\n",
-    "con.sql('''\n",
-    "COPY(\n",
-    "WITH geom1 AS (\n",
-    "  SELECT geom\n",
-    "  FROM ST_Read('new.geojson')\n",
-    "),\n",
-    "geom2 AS (\n",
-    "  SELECT geom\n",
-    "  FROM ST_Read('old.geojson')\n",
-    ")\n",
-    "SELECT ST_Difference(\n",
-    "   (SELECT geom FROM geom1),\n",
-    "   (SELECT geom FROM geom2)\n",
-    ") AS geom\n",
-    ") TO 'simple.geojson' WITH (FORMAT GDAL, DRIVER 'GeoJSON', LAYER_CREATION_OPTIONS 'WRITE_BBOX=YES')\n",
-    "''')\n",
-    "\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "b47fd840-a918-4690-adc0-ca074d09932c",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {