Merge pull request #2 from buuck/develop

Got working version of calculator, no map

.gitignore

@@ -1,3 +1,5 @@
 .venv
 __pycache__/
 service_account.json
+ee_service_account.json
+md_service_token.txt

app.py

@@ -1,17 +1,286 @@
 import gradio as gr
 import plotly.graph_objects as go
-from datasets import load_dataset
+# import ee
+# # import geemap
+
+# # GEE
+# service_account = 'climatebase-july-2023@ee-geospatialml-aquarry.iam.gserviceaccount.com'
+# credentials = ee.ServiceAccountCredentials(service_account, 'service_account.json')
+# ee.Initialize(credentials)
+
+# # Gradio dataset
+# dataset = load_dataset("gradio/NYC-Airbnb-Open-Data", split="train")
+# df = dataset.to_pandas()
+
+import os
+import duckdb
+import pandas as pd
+import datetime
 import ee
 # import geemap
+import yaml
+
+# Define constants
+MD_SERVICE_TOKEN = 'md_service_token.txt'
+# to-do: set-up with papermill parameters
+DATE='2020-01-01'
+YEAR = 2020
+LOCATION=[-74.653370, 5.845328]
+ROI_RADIUS = 20000
+GEE_SERVICE_ACCOUNT = 'climatebase-july-2023@ee-geospatialml-aquarry.iam.gserviceaccount.com'
+GEE_SERVICE_ACCOUNT_CREDENTIALS_FILE = 'ee_service_account.json'
+INDICES_FILE = 'indices.yaml'
+START_YEAR = 2015
+END_YEAR = 2022
+
+class IndexGenerator:
+ """
+ A class to generate indices and compute zonal means.
+
+ Args:
+ centroid (tuple): The centroid coordinates (latitude, longitude) of the region of interest.
+ year (int): The year for which indices are generated.
+ roi_radius (int, optional): The radius (in meters) for creating a buffer around the centroid as the region of interest. Defaults to 20000.
+ project_name (str, optional): The name of the project. Defaults to "".
+ map (geemap.Map, optional): Map object for mapping. Defaults to None (i.e. no map created)
+ """
+ def __init__(self,
+ centroid,
+ roi_radius,
+ year,
+ indices_file,
+ project_name="",
+ map = None,
+ ):
+ self.indices = self._load_indices(indices_file)
+ self.centroid = centroid
+ self.roi = ee.Geometry.Point(*centroid).buffer(roi_radius)
+ self.year = year
+ self.start_date = str(datetime.date(self.year, 1, 1))
+ self.end_date = str(datetime.date(self.year, 12, 31))
+ self.daterange=[self.start_date, self.end_date]
+ self.project_name=project_name
+ self.map = map
+ if self.map is not None:
+ self.show = True
+ else:
+ self.show = False
+
+ def _cloudfree(self, gee_path):
+ """
+ Internal method to generate a cloud-free composite.
+
+ Args:
+ gee_path (str): The path to the Google Earth Engine (GEE) image or image collection.
+
+ Returns:
+ ee.Image: The cloud-free composite clipped to the region of interest.
+ """
+ # Load a raw Landsat ImageCollection for a single year.
+ collection = (
+ ee.ImageCollection(gee_path)
+ .filterDate(*self.daterange)
+ .filterBounds(self.roi)
+ )
+
+ # Create a cloud-free composite with custom parameters for cloud score threshold and percentile.
+ composite_cloudfree = ee.Algorithms.Landsat.simpleComposite(**{
+ 'collection': collection,
+ 'percentile': 75,
+ 'cloudScoreRange': 5
+ })
+ return composite_cloudfree.clip(self.roi)
+
+ def _load_indices(self, indices_file):
+ # Read index configurations
+ with open(indices_file, 'r') as stream:
+ try:
+ return yaml.safe_load(stream)
+ except yaml.YAMLError as e:
+ print(e)
+ return None
+ 
+ def show_map(self, map=None):
+ if map is not None:
+ self.map = map
+ self.show = True
+
+ def disable_map(self):
+ self.show = False
+ 
+ def generate_index(self, index_config):
+ """
+ Generates an index based on the provided index configuration.
+
+ Args:
+ index_config (dict): Configuration for generating the index.
+
+ Returns:
+ ee.Image: The generated index clipped to the region of interest.
+ """
+ match index_config["gee_type"]:
+ case 'image':
+ dataset = ee.Image(index_config['gee_path']).clip(self.roi)
+ if index_config.get('select'):
+ dataset = dataset.select(index_config['select'])
+ case 'image_collection':
+ dataset = ee.ImageCollection(index_config['gee_path']).filterBounds(self.roi).map(lambda image: image.clip(self.roi)).mean()
+ if index_config.get('select'):
+ dataset = dataset.select(index_config['select'])
+ case 'feature_collection':
+ dataset = ee.Image().float().paint(ee.FeatureCollection(index_config['gee_path']), index_config['select']).clip(self.roi)
+ case 'algebraic':
+ image = self._cloudfree(index_config['gee_path'])
+ dataset = image.normalizedDifference(['B4', 'B3'])
+ case _:
+ dataset=None
+
+ if not dataset:
+ raise Exception("Failed to generate dataset.")
+ if self.show and index_config.get('show'):
+ map.addLayer(dataset, index_config['viz'], index_config['name'])
+ print(f"Generated index: {index_config['name']}")
+ return dataset
+
+ def zonal_mean_index(self, index_key):
+ index_config = self.indices[index_key]
+ dataset = self.generate_index(index_config)
+ # zm = self._zonal_mean(single, index_config.get('bandname') or 'constant')
+ out = dataset.reduceRegion(**{
+ 'reducer': ee.Reducer.mean(),
+ 'geometry': self.roi,
+ 'scale': 200 # map scale
+ }).getInfo()
+ if index_config.get('bandname'):
+ return out[index_config.get('bandname')]
+ return out
+
+ def generate_composite_index_df(self, indices=[]):
+ data={
+ "metric": indices,
+ "year":self.year,
+ "centroid": str(self.centroid),
+ "project_name": self.project_name,
+ "value": list(map(self.zonal_mean_index, indices)),
+ "area": roi.area().getInfo(), # m^2
+ "geojson": str(roi.getInfo()),
+ }
+
+ print('data', data)
+ df = pd.DataFrame(data)
+ return df
+
+def set_up_duckdb(service_token_file=None):
+ print('setting up duckdb')
+ # use `climatebase` db
+ if service_token_file is not None:
+ with open(service_token_file, 'r') as f:
+ md_service_token=f.read()
+ 
+ os.environ['motherduck_token'] = md_service_token
+ con = duckdb.connect('md:climatebase')
+ else:
+ con = duckdb.connect(':climatebase:')
+ con.sql("USE climatebase;")
+
+ # load extensions
+ con.sql("""INSTALL spatial; LOAD spatial;""")
+
+ return con
+ 
+def authenticate_gee(gee_service_account, gee_service_account_credentials_file):
+ print('authenticate_gee')
+ # to-do: alert if dataset filter date nan
+ credentials = ee.ServiceAccountCredentials(gee_service_account, gee_service_account_credentials_file)
+ ee.Initialize(credentials)
+ 
+def load_indices(indices_file):
+ # Read index configurations
+ with open(indices_file, 'r') as stream:
+ try:
+ return yaml.safe_load(stream)
+ except yaml.YAMLError as e:
+ print(e)
+ return None
 
-# GEE
-service_account = 'climatebase-july-2023@ee-geospatialml-aquarry.iam.gserviceaccount.com'
-credentials = ee.ServiceAccountCredentials(service_account, 'service_account.json')
-ee.Initialize(credentials)
+def create_dataframe(years, project_name):
+ dfs=[]
+ print(years)
+ indices = load_indices(INDICES_FILE)
+ for year in years:
+ print(year)
+ ig = IndexGenerator(centroid=LOCATION, roi_radius=ROI_RADIUS, year=year, indices_file=INDICES_FILE, project_name=project_name)
+ df = ig.generate_composite_index_df(list(indices.keys()))
+ dfs.append(df)
+ return pd.concat(dfs)
 
-# Gradio dataset
-dataset = load_dataset("gradio/NYC-Airbnb-Open-Data", split="train")
-df = dataset.to_pandas()
+# def preview_table():
+# con.sql("FROM bioindicator;").show()
+
+# if __name__ == '__main__':
+
+
+ # Map = geemap.Map()
+
+
+ # # Create a cloud-free composite with custom parameters for cloud score threshold and percentile.
+ # composite_cloudfree = ee.Algorithms.Landsat.simpleComposite(**{
+ # 'collection': collection,
+ # 'percentile': 75,
+ # 'cloudScoreRange': 5
+ # })
+
+ # Map.addLayer(composite_cloudfree, {'bands': ['B4', 'B3', 'B2'], 'max': 128}, 'Custom TOA composite')
+ # Map.centerObject(roi, 14)
+
+
+ # ig = IndexGenerator(centroid=LOCATION, year=2015, indices_file=INDICES_FILE, project_name='Test Project', map=Map)
+ # dataset = ig.generate_index(indices['Air'])
+
+ # minMax = dataset.clip(roi).reduceRegion(
+ # geometry = roi,
+ # reducer = ee.Reducer.minMax(),
+ # scale= 3000,
+ # maxPixels= 10e3,
+ # )
+
+ # minMax.getInfo()
+def calculate_biodiversity_score(start_year, end_year, project_name):
+ years = []
+ for year in range(start_year, end_year):
+ row_exists = con.sql(f"SELECT COUNT(1) FROM bioindicator WHERE (year = {year} AND project_name = '{project_name}')").fetchall()[0][0]
+ if not row_exists:
+ years.append(year)
+
+ if len(years)>0:
+ df = create_dataframe(years, project_name)
+ # con.sql('FROM df LIMIT 5').show()
+
+ # Write score table to `_temptable`
+ con.sql('CREATE OR REPLACE TABLE _temptable AS SELECT *, (value * area) AS score FROM (SELECT year, project_name, AVG(value) AS value, area FROM df GROUP BY year, project_name, area ORDER BY project_name)')
+
+
+ # Create `bioindicator` table IF NOT EXISTS.
+ con.sql("""
+ USE climatebase;
+ CREATE TABLE IF NOT EXISTS bioindicator (year BIGINT, project_name VARCHAR(255), value DOUBLE, area DOUBLE, score DOUBLE, CONSTRAINT unique_year_project_name UNIQUE (year, project_name));
+ """)
+
+ return con.sql(f"SELECT * FROM bioindicator WHERE (year > {start_year} AND year <= {end_year} AND project_name = '{project_name}')").df()
+
+def view_all():
+ print('view_all')
+ return con.sql(f"SELECT * FROM bioindicator").df()
+
+def push_to_md():
+ # UPSERT project record
+ con.sql("""
+ INSERT INTO bioindicator FROM _temptable
+ ON CONFLICT (year, project_name) DO UPDATE SET value = excluded.value;
+ """)
+ print('Saved records')
+
+# preview_table()
 
 def filter_map(min_price, max_price, boroughs):
 
@@ -49,16 +318,43 @@ def filter_map(min_price, max_price, boroughs):
  return fig
 
 with gr.Blocks() as demo:
+ con = set_up_duckdb(MD_SERVICE_TOKEN)
+ authenticate_gee(GEE_SERVICE_ACCOUNT, GEE_SERVICE_ACCOUNT_CREDENTIALS_FILE)
+ # Create circle buffer over point
+ # roi = ee.Geometry.Point(*LOCATION).buffer(ROI_RADIUS)
+
+ # # Load a raw Landsat ImageCollection for a single year.
+ # start_date = str(datetime.date(YEAR, 1, 1))
+ # end_date = str(datetime.date(YEAR, 12, 31))
+ # collection = (
+ # ee.ImageCollection('LANDSAT/LC08/C02/T1')
+ # .filterDate(start_date, end_date)
+ # .filterBounds(roi)
+ # )
+
+ # indices = load_indices(INDICES_FILE)
+ # push_to_md(START_YEAR, END_YEAR, 'Test Project')
  with gr.Column():
+ # map = gr.Plot().style()
+ with gr.Row():
+ start_year = gr.Number(value=2017, label="Start Year", precision=0)
+ end_year = gr.Number(value=2022, label="End Year", precision=0)
+ project_name = gr.Textbox(label='Project Name')
+ # boroughs = gr.CheckboxGroup(choices=["Queens", "Brooklyn", "Manhattan", "Bronx", "Staten Island"], value=["Queens", "Brooklyn"], label="Select Methodology:")
+ # btn = gr.Button(value="Update Filter")
  with gr.Row():
- min_price = gr.Number(value=250, label="Project Name")
- max_price = gr.Number(value=1000, label="Project Description")
- boroughs = gr.CheckboxGroup(choices=["Queens", "Brooklyn", "Manhattan", "Bronx", "Staten Island"], value=["Queens", "Brooklyn"], label="Select Methodology:")
- btn = gr.Button(value="Update Filter")
- btn = gr.Button(value="Save")
- btn = gr.Button(value="Run")
- map = gr.Plot().style()
- demo.load(filter_map, [min_price, max_price, boroughs], map)
- btn.click(filter_map, [min_price, max_price, boroughs], map)
+ calc_btn = gr.Button(value="Calculate!")
+ view_btn = gr.Button(value="View all")
+ save_btn = gr.Button(value="Save")
+ results_df = gr.Dataframe(
+ headers=["Year", "Project Name", "Score"],
+ datatype=["number", "str", "number"],
+ label="Biodiversity scores by year",
+ )
+ # demo.load(filter_map, [min_price, max_price, boroughs], map)
+ # btn.click(filter_map, [min_price, max_price, boroughs], map)
+ calc_btn.click(calculate_biodiversity_score, inputs=[start_year, end_year, project_name], outputs=results_df)
+ view_btn.click(view_all, outputs=results_df)
+ save_btn.click(push_to_md)
 
 demo.launch()

conda_environment.yml

@@ -0,0 +1,17 @@
+name: openbiodiversity_calculator
+channels:
+ - conda-forge
+ - plotly
+dependencies:
+ - earthengine-api
+ - geemap
+ - geopandas
+ - plotly
+ - segment-geospatial
+ - pandas
+ - pip
+ - pip:
+ - duckdb==0.8.1
+ - geojson
+ - gradio
+ - pyyaml

indices.yaml

@@ -0,0 +1,169 @@
+---
+Water:
+ name: Water
+ roi: ''
+ gee_path: JRC/GSW1_1/GlobalSurfaceWater
+ gee_type: image
+ viz:
+ min: 0
+ max: 100
+ palette:
+ - ffffff
+ - ffbbbb
+ - 0000ff
+ bandname: occurrence
+ select: occurrence
+ show: true
+Protected:
+ name: Protected
+ roi: ''
+ gee_path: WCMC/WDPA/current/polygons
+ gee_type: feature_collection
+ viz:
+ palette:
+ - 2ed033
+ - 5aff05
+ - 67b9ff
+ - 5844ff
+ - 0a7618
+ - 2c05ff
+ min: 0
+ max: 1550000
+ opacity: 0.8
+ select: REP_AREA
+ bandname: constant
+ show: true
+Air:
+ name: Air
+ roi: ''
+ gee_path: COPERNICUS/S5P/OFFL/L3_AER_AI
+ gee_type: image_collection
+ viz:
+ min: -1
+ max: 2
+ palette:
+ - black
+ - blue
+ - purple
+ - cyan
+ - green
+ - yellow
+ - red
+ bandname: absorbing_aerosol_index
+ select: absorbing_aerosol_index
+ dates: false
+ show: false
+Soil:
+ name: Soil
+ roi: ''
+ gee_path: OpenLandMap/SOL/SOL_ORGANIC-CARBON_USDA-6A1C_M/v02
+ gee_type: image
+ viz:
+ bands:
+ - b200
+ min: 0
+ max: 12
+ palette:
+ - ffffa0
+ - f7fcb9
+ - d9f0a3
+ - addd8e
+ - 78c679
+ - 41ab5d
+ - '238443'
+ - 005b29
+ - 004b29
+ - 012b13
+ - 00120b
+ select: b0
+ bandname: b0
+ show: false
+Temperature:
+ name: Temperature
+ roi: ''
+ gee_path: MODIS/061/MYD21C1
+ gee_type: image_collection
+ viz:
+ min: 216
+ max: 348
+ palette:
+ - '040274'
+ - '040281'
+ - 0502a3
+ - 0502b8
+ - 0502ce
+ - 0502e6
+ - 0602ff
+ - 235cb1
+ - 307ef3
+ - 269db1
+ - 30c8e2
+ - 32d3ef
+ - 3be285
+ - 3ff38f
+ - 86e26f
+ - 3ae237
+ - b5e22e
+ - d6e21f
+ - fff705
+ - ffd611
+ - ffb613
+ - ff8b13
+ - ff6e08
+ - ff500d
+ - ff0000
+ - de0101
+ - c21301
+ - a71001
+ - '911003'
+ select: LST_Day
+ bandname: LST_Day
+ dates: true
+ show: true
+Habitat:
+ name: Habitat
+ roi: ''
+ gee_path: projects/sat-io/open-datasets/IUCN_HABITAT/iucn_habitatclassification_composite_lvl2_ver004
+ gee_type: image
+ viz: {}
+ bandname: comp_first
+ show: true
+NDVI:
+ name: NDVI
+ roi: ''
+ gee_path: LANDSAT/LC08/C02/T1
+ gee_type: algebraic
+ normalized_difference:
+ - B4
+ - B3
+ viz:
+ palette:
+ - "#d73027"
+ - "#f46d43"
+ - "#fdae61"
+ - "#fee08b"
+ - "#d9ef8b"
+ - "#a6d96a"
+ - "#66bd63"
+ - "#1a9850"
+ bandname: nd
+NDWI:
+ name: NDWI
+ roi: ''
+ gee_path: LANDSAT/LC08/C02/T1
+ gee_type: algebraic
+ normalized_difference:
+ - B3
+ - B5
+ viz:
+ palette:
+ - "#ece7f2"
+ - "#d0d1e6"
+ - "#a6bddb"
+ - "#74a9cf"
+ - "#3690c0"
+ - "#0570b0"
+ - "#045a8d"
+ - "#023858"
+ bandname: nd
+ show: true