Added coefficients/normalizations. Added rounding.

app.py

@@ -1,3 +1,4 @@
+import os
 import gradio as gr
 
 from utils import duckdb_queries as dq
@@ -5,13 +6,18 @@ from utils.gradio import get_window_url_params
 from utils.indicators import IndexGenerator
 
 # Instantiate outside gradio app to avoid re-initializing GEE, which is slow
-indexgenerator = IndexGenerator(indices=["NDWI", "Water", "Protected", "Habitat"])
+indexgenerator = IndexGenerator()
+
+metric_names = os.listdir('metrics')
+for i in range(len(metric_names)):
+ metric_names[i] = metric_names[i].split('.yaml')[0].replace('_', ' ')
 
 with gr.Blocks() as demo:
  with gr.Column():
  m1 = gr.Plot()
  with gr.Row():
  project_name = gr.Dropdown([], label="Project", value="Select project")
+ metric = gr.Dropdown(metric_names, label='Metric', value='Select metric')
  start_year = gr.Number(value=2017, label="Start Year", precision=0)
  end_year = gr.Number(value=2022, label="End Year", precision=0)
  with gr.Row():
@@ -37,8 +43,7 @@ with gr.Blocks() as demo:
  projects = dq.list_projects_by_author(author_id=username)
  # Initialize the first project in the list
  project_names = projects['name'].tolist()
- default_project = project_names[0]
- return gr.Dropdown.update(choices=project_names, value=default_project)
+ return gr.Dropdown.update(choices=project_names)
 
  # Change the project name in the index generator object when the
  # user selects a new project
@@ -47,6 +52,12 @@ with gr.Blocks() as demo:
  inputs=project_name
  )
 
+ # Set the metric to be calculated
+ metric.change(
+ indexgenerator.set_metric,
+ inputs=metric
+ )
+
  # Get url params
  url_params = gr.JSON({"username": "default"}, visible=False, label="URL Params")
 

metrics/OE_Biodiversity.yaml

@@ -0,0 +1,109 @@
+---
+Protected:
+ name: Protected
+ description: The total PA extent, including both marine (if applicable) and terrestrial areas provided by data provider as specified in the legal text for the site.
+ unit: km^2
+ min: 0
+ max: roi_area
+ roi: ''
+ gee_path: WCMC/WDPA/current/polygons
+ gee_type: feature_collection
+ viz:
+ palette:
+ - 2ed033
+ - 5aff05
+ - 67b9ff
+ - 5844ff
+ - 0a7618
+ - 2c05ff
+ min: 0
+ max: 1550000
+ opacity: 0.8
+ select: REP_AREA
+ bandname: constant
+ coefficient: 1
+ show: true
+Soil:
+ name: Soil
+ description: An estimate of soil organic carbon content at b{n} cm depth.
+ unit: 5g/kg
+ min: 0
+ max: 120
+ roi: ''
+ gee_path: OpenLandMap/SOL/SOL_ORGANIC-CARBON_USDA-6A1C_M/v02
+ gee_type: image
+ viz:
+ bands:
+ - b0
+ min: 0
+ max: 12
+ palette:
+ - ffffa0
+ - f7fcb9
+ - d9f0a3
+ - addd8e
+ - 78c679
+ - 41ab5d
+ - '238443'
+ - 005b29
+ - 004b29
+ - 012b13
+ - 00120b
+ select: b0
+ bandname: b0
+ coefficient: 1
+ show: false
+NDVI:
+ name: NDVI
+ description: Normalized difference vegetation index
+ unit: index (continuous)
+ min: -1
+ max: 1
+ roi: ''
+ gee_path: LANDSAT/LC08/C02/T1
+ gee_type: algebraic
+ normalized_difference:
+ - B4
+ - B3
+ viz:
+ min: -1
+ max: 1
+ palette:
+ - "#d73027"
+ - "#f46d43"
+ - "#fdae61"
+ - "#fee08b"
+ - "#d9ef8b"
+ - "#a6d96a"
+ - "#66bd63"
+ - "#1a9850"
+ bandname: nd
+ coefficient: 1
+ show: true
+NDWI:
+ name: NDWI
+ description: An estimate of the water content of leaves.
+ unit: index (continuous)
+ min: -1
+ max: 1
+ roi: ''
+ gee_path: LANDSAT/LC08/C02/T1
+ gee_type: algebraic
+ normalized_difference:
+ - B5
+ - B6
+ viz:
+ min: -1
+ max: 1
+ palette:
+ - "#ece7f2"
+ - "#d0d1e6"
+ - "#a6bddb"
+ - "#74a9cf"
+ - "#3690c0"
+ - "#0570b0"
+ - "#045a8d"
+ - "#023858"
+ bandname: nd
+ coefficient: 1
+ show: true

indices.yaml → unused_metrics/OE_Biodiversity_20230723.yaml

@@ -1,6 +1,10 @@
 ---
 Water:
  name: Water
+ description: The percent of a given area covered with water.
+ unit: percentage
+ min: 0
+ max: 100
  roi: ''
  gee_path: JRC/GSW1_1/GlobalSurfaceWater
  gee_type: image
@@ -13,9 +17,14 @@ Water:
  - 0000ff
  bandname: occurrence
  select: occurrence
+ coefficient: 1
  show: true
 Protected:
  name: Protected
+ description: The total PA extent, including both marine (if applicable) and terrestrial areas provided by data provider as specified in the legal text for the site.
+ unit: km^2
+ min: 0
+ max: roi_area
  roi: ''
  gee_path: WCMC/WDPA/current/polygons
  gee_type: feature_collection
@@ -32,9 +41,14 @@ Protected:
  opacity: 0.8
  select: REP_AREA
  bandname: constant
+ coefficient: 1
  show: true
 Air:
  name: Air
+ description: A measure of the prevalence of aerosols in the atmosphere.
+ unit: index (continuous)
+ min: -21
+ max: 39
  roi: ''
  gee_path: COPERNICUS/S5P/OFFL/L3_AER_AI
  gee_type: image_collection
@@ -52,15 +66,20 @@ Air:
  bandname: absorbing_aerosol_index
  select: absorbing_aerosol_index
  dates: false
+ coefficient: 1
  show: false
 Soil:
  name: Soil
+ description: An estimate of soil organic carbon content at b{n} cm depth.
+ unit: 5g/kg
+ min: 0
+ max: 120
  roi: ''
  gee_path: OpenLandMap/SOL/SOL_ORGANIC-CARBON_USDA-6A1C_M/v02
  gee_type: image
  viz:
  bands:
- - b200
+ - b0
  min: 0
  max: 12
  palette:
@@ -77,9 +96,14 @@ Soil:
  - 00120b
  select: b0
  bandname: b0
+ coefficient: 1
  show: false
 Temperature:
  name: Temperature
+ description: Average Daytime Land Surface Temperature
+ unit: 0.02 K
+ min: 7500
+ max: 65535
  roi: ''
  gee_path: MODIS/061/MYD21C1
  gee_type: image_collection
@@ -119,17 +143,24 @@ Temperature:
  select: LST_Day
  bandname: LST_Day
  dates: true
+ coefficient: 1
  show: true
 Habitat:
  name: Habitat
+ unit: Classification index (categorical)
  roi: ''
  gee_path: projects/sat-io/open-datasets/IUCN_HABITAT/iucn_habitatclassification_composite_lvl2_ver004
  gee_type: image
  viz: {}
  bandname: comp_first
+ coefficient: 1
  show: true
 NDVI:
  name: NDVI
+ description: Normalized difference vegetation index
+ unit: index (continuous)
+ min: -1
+ max: 1
  roi: ''
  gee_path: LANDSAT/LC08/C02/T1
  gee_type: algebraic
@@ -137,6 +168,8 @@ NDVI:
  - B4
  - B3
  viz:
+ min: -1
+ max: 1
  palette:
  - "#d73027"
  - "#f46d43"
@@ -147,15 +180,23 @@ NDVI:
  - "#66bd63"
  - "#1a9850"
  bandname: nd
+ coefficient: 1
+ show: true
 NDWI:
  name: NDWI
+ description: An estimate of the water content of leaves.
+ unit: index (continuous)
+ min: -1
+ max: 1
  roi: ''
  gee_path: LANDSAT/LC08/C02/T1
  gee_type: algebraic
  normalized_difference:
- - B3
  - B5
+ - B6
  viz:
+ min: -1
+ max: 1
  palette:
  - "#ece7f2"
  - "#d0d1e6"
@@ -166,4 +207,5 @@ NDWI:
  - "#045a8d"
  - "#023858"
  bandname: nd
+ coefficient: 1
  show: true

utils/duckdb_queries.py

@@ -46,6 +46,12 @@ def get_project_scores(project_name, start_year, end_year):
  ).df()
 
 
+def check_if_table_exists(table_name):
+ tables = con.execute("SHOW TABLES;").fetchall()
+ for i in range(len(tables)):
+ tables[i] = tables[i][0]
+ return table_name in tables
+
 def check_if_project_exists_for_year(project_name, year):
  return con.execute(
  "SELECT COUNT(1) FROM bioindicator WHERE (year = ? AND project_name = ?)",
@@ -55,7 +61,7 @@ def check_if_project_exists_for_year(project_name, year):
 
 def write_score_to_temptable(df):
  con.sql(
- "CREATE OR REPLACE TABLE _temptable AS SELECT *, ROUND((value * area), 2) AS score FROM (SELECT year, project_name, ROUND(AVG(value), 2) AS value, area FROM df GROUP BY year, project_name, area ORDER BY project_name)"
+ "CREATE OR REPLACE TABLE _temptable AS SELECT *, (value * area) AS score FROM (SELECT year, project_name, metric, AVG(value * coefficient) AS value, area FROM df GROUP BY year, project_name, metric, area ORDER BY project_name, metric)"
  )
  return True
 
@@ -64,7 +70,7 @@ def get_or_create_bioindicator_table():
  con.sql(
  """
  USE climatebase;
- CREATE TABLE IF NOT EXISTS bioindicator (year BIGINT, project_name VARCHAR(255), value DOUBLE, area DOUBLE, score DOUBLE, CONSTRAINT unique_year_project_name UNIQUE (year, project_name));
+ CREATE TABLE IF NOT EXISTS bioindicator (year BIGINT, project_name VARCHAR(255), metric VARCHAR(255), value DOUBLE, area DOUBLE, score DOUBLE, CONSTRAINT unique_year_project_name_metric UNIQUE (year, project_name, metric));
  """
  )
  return True
@@ -74,7 +80,7 @@ def upsert_project_record():
  con.sql(
  """
  INSERT INTO bioindicator FROM _temptable
- ON CONFLICT (year, project_name) DO UPDATE SET value = excluded.value;
+ ON CONFLICT (year, project_name, metric) DO UPDATE SET value = excluded.value;
  """
  )
  return True

utils/indicators.py

@@ -16,8 +16,6 @@ from . import logging
 GEE_SERVICE_ACCOUNT = (
  "climatebase-july-2023@ee-geospatialml-aquarry.iam.gserviceaccount.com"
 )
-INDICES_FILE = "indices.yaml"
-
 
 class IndexGenerator:
  """
@@ -27,20 +25,22 @@ class IndexGenerator:
  indices (string[], required): Array of index names to include in aggregate index generation.
  """
 
- def __init__(
- self,
- indices,
- ):
+ def __init__(self):
  # Authenticate to GEE & DuckDB
  self._authenticate_ee(GEE_SERVICE_ACCOUNT)
 
+ self.roi = None
  self.project_name = None
  self.project_geometry = None
  self.project_centroid = None
- 
+ self.indices = None
+ self.metric_name = None
+
+ def set_metric(self, metric_name): 
  # Use defined subset of indices
- all_indices = self._load_indices(INDICES_FILE)
- self.indices = {k: all_indices[k] for k in indices}
+ indices_file = f'metrics/{metric_name.replace(" ", "_")}.yaml'
+ self.indices = self._load_indices(indices_file)
+ self.metric_name = metric_name
 
  def set_project(self, project_name):
  self.project_name = project_name
@@ -139,6 +139,18 @@ class IndexGenerator:
  if not dataset:
  raise Exception("Failed to generate dataset.")
 
+ # Normalize to a range of [0, 1]
+ min_val = 0
+ max_val = 1
+ if type(index_config['min'])==int or type(index_config['min']==float):
+ min_val = index_config['min']
+ if str(index_config['max'])=='roi_area':
+ max_val = self.roi.area().getInfo() # in m^2
+ elif type(index_config['max'])==int or type(index_config['max']==float):
+ max_val = index_config['max']
+ dataset.subtract(min_val)\
+ .divide(max_val - min_val)
+
  logging.info(f"Generated index: {index_config['name']}")
  return dataset
 
@@ -163,17 +175,17 @@ class IndexGenerator:
  logging.info(f"Calculated zonal mean for {index_key}.")
  return out
 
- def generate_composite_index_df(self, year, indices=[]):
+ def generate_composite_index_df(self, year):
  data = {
- "metric": indices,
+ "metric": self.metric_name,
  "year": year,
  "centroid": "",
  "project_name": "",
- "value": list(map(self.zonal_mean_index, indices, repeat(year))),
+ "value": list(map(self.zonal_mean_index, self.indices, repeat(year))),
  # to-do: calculate with duckdb; also, should be part of project table instead
  "area": self.roi.area().getInfo(), # m^2
  "geojson": "",
- # to-do: coefficient
+ "coefficient": list(map(lambda x: self.indices[x]['coefficient'], self.indices))
  }
 
  logging.info("data", data)
@@ -199,9 +211,7 @@ class IndexGenerator:
  # to-do: pararelize?
  for year in years:
  logging.info(year)
- df = self.generate_composite_index_df(
- year, self.project_geometry, list(self.indices.keys())
- )
+ df = self.generate_composite_index_df(year)
  dfs.append(df)
 
  # Concatenate all dataframes
@@ -299,7 +309,9 @@ class IndexGenerator:
 
  def calculate_score(self, start_year, end_year):
  years = []
- for year in range(start_year, end_year):
+ # Create `bioindicator` table IF NOT EXISTS.
+ dq.get_or_create_bioindicator_table()
+ for year in range(start_year, end_year+1):
  row_exists = dq.check_if_project_exists_for_year(self.project_name, year)
  if not row_exists:
  years.append(year)
@@ -310,11 +322,20 @@ class IndexGenerator:
  # Write score table to `_temptable`
  dq.write_score_to_temptable(df)
 
- # Create `bioindicator` table IF NOT EXISTS.
- dq.get_or_create_bioindicator_table()
-
  # UPSERT project record
  dq.upsert_project_record()
  logging.info("upserted records into motherduck")
  scores = dq.get_project_scores(self.project_name, start_year, end_year)
+ scores.columns = scores.columns.str.replace('_', ' ').str.title()
+ if 'Area' in scores.columns:
+ scores['Area'] /= 1000**2
+ scores.rename(columns={'Area':'Area (km^2)'}, inplace=True)
+ if 'Score' in scores.columns:
+ scores['Score'] /= 1000**2
+ scores.rename(columns={'Score': 'Score (Area * Value)'}, inplace=True)
+ # Round scores to 4 significant figures
+ scores = scores.apply(
+ lambda x: ['%.4g'%x_i for x_i in x]
+ if pd.api.types.is_numeric_dtype(x)
+ else x)
  return scores