added conversion tests

.github/workflows/docker-hub.yml

@@ -23,9 +23,9 @@ jobs:
  - name: Build Docker image
  run: |
  docker build \
- -t joshuasundance/geospatial-data-converter:${{ github.ref_name }} \
- -t joshuasundance/geospatial-data-converter:latest \
+ -t joshuasundance/geospatial_data_converter:${{ github.ref_name }} \
+ -t joshuasundance/geospatial_data_converter:latest \
  .
 
  - name: Push to Docker Hub
- run: docker push -a joshuasundance/geospatial-data-converter
+ run: docker push -a joshuasundance/geospatial_data_converter

.github/workflows/hf-space.yml

@@ -18,4 +18,4 @@ jobs:
  env:
  HF_TOKEN: ${{ secrets.HF_TOKEN }}
  run: |
- git push https://joshuasundance:[email protected]/spaces/joshuasundance/geospatial-data-converter main
+ git push https://joshuasundance:[email protected]/spaces/joshuasundance/geospatial_data_converter main

Dockerfile

@@ -12,9 +12,9 @@ COPY ./requirements.txt /home/appuser/requirements.txt
 RUN pip install --user --no-cache-dir --upgrade -r /home/appuser/requirements.txt
 
 COPY geospatial_data_converter/ /home/appuser/geospatial_data_converter/
+COPY tests/ /home/appuser/tests/
 
 WORKDIR /workspace
 EXPOSE 7860
 
 CMD ["streamlit", "run", "/home/appuser/geospatial_data_converter/app.py", "--server.port", "7860", "--server.address", "0.0.0.0", "--server.enableXsrfProtection=false"]
-#CMD ["/bin/bash"]

README.md

@@ -1,5 +1,5 @@
 ---
-title: geospatial-data-converter
+title: geospatial_data_converter
 emoji: 🌎
 colorFrom: green
 colorTo: blue
@@ -9,21 +9,21 @@ pinned: true
 tags: [geospatial, streamlit, docker]
 ---
 
-# geospatial-data-converter
+# geospatial_data_converter
 
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![python](https://img.shields.io/badge/Python-3.11-3776AB.svg?style=flat&logo=python&logoColor=white)](https://www.python.org)
-![GitHub tag (with filter)](https://img.shields.io/github/v/tag/joshuasundance-swca/geospatial-data-converter)
+![GitHub tag (with filter)](https://img.shields.io/github/v/tag/joshuasundance-swca/geospatial_data_converter)
 
-[![Push to Docker Hub](https://github.com/joshuasundance-swca/geospatial-data-converter/actions/workflows/docker-hub.yml/badge.svg)](https://github.com/joshuasundance-swca/geospatial-data-converter/actions/workflows/docker-hub.yml)
-[![Docker Image Size (tag)](https://img.shields.io/docker/image-size/joshuasundance/geospatial-data-converter/latest)](https://hub.docker.com/r/joshuasundance/geospatial-data-converter)
+[![Push to Docker Hub](https://github.com/joshuasundance-swca/geospatial_data_converter/actions/workflows/docker-hub.yml/badge.svg)](https://github.com/joshuasundance-swca/geospatial_data_converter/actions/workflows/docker-hub.yml)
+[![Docker Image Size (tag)](https://img.shields.io/docker/image-size/joshuasundance/geospatial_data_converter/latest)](https://hub.docker.com/r/joshuasundance/geospatial_data_converter)
 
-[![Push to HuggingFace Space](https://github.com/joshuasundance-swca/geospatial-data-converter/actions/workflows/hf-space.yml/badge.svg)](https://github.com/joshuasundance-swca/geospatial-data-converter/actions/workflows/hf-space.yml)
-[![Open HuggingFace Space](https://huggingface.co/datasets/huggingface/badges/raw/main/open-in-hf-spaces-sm.svg)](https://huggingface.co/spaces/joshuasundance/geospatial-data-converter)
+[![Push to HuggingFace Space](https://github.com/joshuasundance-swca/geospatial_data_converter/actions/workflows/hf-space.yml/badge.svg)](https://github.com/joshuasundance-swca/geospatial_data_converter/actions/workflows/hf-space.yml)
+[![Open HuggingFace Space](https://huggingface.co/datasets/huggingface/badges/raw/main/open-in-hf-spaces-sm.svg)](https://huggingface.co/spaces/joshuasundance/geospatial_data_converter)
 
-![Code Climate maintainability](https://img.shields.io/codeclimate/maintainability/joshuasundance-swca/geospatial-data-converter)
-![Code Climate issues](https://img.shields.io/codeclimate/issues/joshuasundance-swca/geospatial-data-converter)
-![Code Climate technical debt](https://img.shields.io/codeclimate/tech-debt/joshuasundance-swca/geospatial-data-converter)
+![Code Climate maintainability](https://img.shields.io/codeclimate/maintainability/joshuasundance-swca/geospatial_data_converter)
+![Code Climate issues](https://img.shields.io/codeclimate/issues/joshuasundance-swca/geospatial_data_converter)
+![Code Climate technical debt](https://img.shields.io/codeclimate/tech-debt/joshuasundance-swca/geospatial_data_converter)
 
 [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit)
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v1.json)](https://github.com/charliermarsh/ruff)
@@ -31,7 +31,7 @@ tags: [geospatial, streamlit, docker]
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 
 [![security: bandit](https://img.shields.io/badge/security-bandit-yellow.svg)](https://github.com/PyCQA/bandit)
-![Known Vulnerabilities](https://snyk.io/test/github/joshuasundance-swca/geospatial-data-converter/badge.svg)
+![Known Vulnerabilities](https://snyk.io/test/github/joshuasundance-swca/geospatial_data_converter/badge.svg)
 
 This project showcases a simple geospatial data converter using [Streamlit](https://streamlit.io) and [GeoPandas](https://geopandas.org/).
 
@@ -45,11 +45,11 @@ This project showcases a simple geospatial data converter using [Streamlit](http
 - Download button for the converted data
 
 # Deployment
-`geospatial-data-converter` is deployed as a [Docker image](https://hub.docker.com/r/<your-dockerhub-username>/geospatial-data-converter) based on the `python:3.11-slim-bookworm` image.
+`geospatial_data_converter` is deployed as a [Docker image](https://hub.docker.com/r/<your-dockerhub-username>/geospatial_data_converter) based on the `python:3.11-slim-bookworm` image.
 
 ## With Docker (pull from Docker Hub)
 1. Run in terminal:
-`docker run -p 7860:7860 <your-dockerhub-username>/geospatial-data-converter:latest`
+`docker run -p 7860:7860 <your-dockerhub-username>/geospatial_data_converter:latest`
 2. Open http://localhost:8501 in your browser
 
 ## Docker Compose (build locally)
@@ -60,7 +60,7 @@ This project showcases a simple geospatial data converter using [Streamlit](http
 ## Kubernetes
 1. Clone the repo. Navigate to cloned repo directory
 2. Run bash script: `/bin/bash ./kubernetes/deploy.sh`
-3. Get the IP address for your new service: `kubectl get service geospatial-data-converter`
+3. Get the IP address for your new service: `kubectl get service geospatial_data_converter`
 
 # Links
 - [Streamlit](https://streamlit.io)

bumpver.toml

@@ -14,5 +14,5 @@ push = true
 "bumpver.toml" = [
  'current_version = "{version}"',
 ]
-"geospatial-data-converter/app.py" = ['__version__ = "{version}"']
-"kubernetes/resources.yaml" = [' image: joshuasundance/geospatial-data-converter:{version}']
+"geospatial_data_converter/app.py" = ['__version__ = "{version}"']
+"kubernetes/resources.yaml" = [' image: joshuasundance/geospatial_data_converter:{version}']

docker-compose.yml

@@ -1,8 +1,8 @@
 version: '3.8'
 
 services:
- geospatial-data-converter:
- image: geospatial-data-converter:latest
+ geospatial_data_converter:
+ image: geospatial_data_converter:latest
  build: .
  ports:
  - "${APP_PORT:-7860}:${APP_PORT:-7860}"
@@ -14,3 +14,8 @@ services:
  "--server.address", "0.0.0.0",
  "--server.enableXsrfProtection=false"
  ]
+ test:
+ image: geospatial_data_converter:latest # Use the same image as your main service
+ build: .
+ working_dir: /home/appuser
+ command: ["/bin/bash"]

geospatial_data_converter/app.py

@@ -6,7 +6,10 @@ import streamlit as st
 from aiohttp import ClientSession
 from restgdf import FeatureLayer
 
-from utils import read_file, convert, output_format_dict
+try:
+ from utils import read_file, convert, output_format_dict
+except ImportError:
+ from .utils import read_file, convert, output_format_dict
 
 __version__ = "0.1.1"
 

geospatial_data_converter/kml_tricks.py

@@ -131,7 +131,7 @@ def extract_data_from_kml_code(kml_code: str) -> pd.DataFrame:
  """Extracts data from KML code into a DataFrame using SimpleData tags, excluding embedded tables in feature descriptions"""
 
  # Parse the KML source code
- soup = bs4.BeautifulSoup(kml_code, "html.parser")
+ soup = bs4.BeautifulSoup(kml_code, features="xml")
 
  # Find all SchemaData tags (representing rows)
  schema_data_tags = soup.find_all("schemadata")

geospatial_data_converter/tests/test_kml.py

@@ -1,16 +0,0 @@
-import sys
-
-sys.path.append("..")
-from utils import convert, read_file
-
-
-def test_kml_coversion():
- with open("boundary.kml", "rb") as f:
- kml = read_file(f)
- converted_data = convert(kml, "boundary_converted.kml", "kml")
- with open("boundary_converted.kml", "wb") as f:
- f.write(converted_data)
-
-
-def test_kmz_conversion():
- pass

geospatial_data_converter/utils.py

@@ -5,7 +5,10 @@ from tempfile import TemporaryDirectory
 from typing import BinaryIO
 import geopandas as gpd
 
-from kml_tricks import load_ge_data
+try:
+ from kml_tricks import load_ge_data
+except ImportError:
+ from .kml_tricks import load_ge_data
 
 output_format_dict = {
  "ESRI Shapefile": ("shp", "zip", "application/zip"), # must be zipped
@@ -22,7 +25,7 @@ def read_file(file: BinaryIO, *args, **kwargs) -> gpd.GeoDataFrame:
  ext = ext.lower().strip(".")
  if ext == "zip":
  with TemporaryDirectory() as tmp_dir:
- tmp_file_path = os.path.join(tmp_dir, file.name)
+ tmp_file_path = os.path.join(tmp_dir, f"{basename}.{ext}")
  with open(tmp_file_path, "wb") as tmp_file:
  tmp_file.write(file.read())
  return gpd.read_file(
@@ -33,7 +36,10 @@ def read_file(file: BinaryIO, *args, **kwargs) -> gpd.GeoDataFrame:
  )
  elif ext in ("kml", "kmz"):
  with TemporaryDirectory() as tmp_dir:
- tmp_file_path = os.path.join(tmp_dir, file.name)
+ tmp_file_path = os.path.join(tmp_dir, f"{basename}.{ext}")
+ print(file.name)
+ print(basename)
+ print(tmp_file_path)
  with open(tmp_file_path, "wb") as tmp_file:
  tmp_file.write(file.read())
  return load_ge_data(tmp_file_path)
@@ -60,7 +66,6 @@ def convert(gdf: gpd.GeoDataFrame, output_name: str, output_format: str) -> byte
  """Convert a GeoDataFrame to the specified format"""
  with TemporaryDirectory() as tmpdir:
  out_path = os.path.join(tmpdir, output_name)
-
  if output_format == "CSV":
  gdf.to_csv(out_path)
  else:

kubernetes/resources.yaml

@@ -1,20 +1,20 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
- name: geospatial-data-converter-deployment
+ name: geospatial_data_converter-deployment
 spec:
  replicas: 1
  selector:
  matchLabels:
- app: geospatial-data-converter
+ app: geospatial_data_converter
  template:
  metadata:
  labels:
- app: geospatial-data-converter
+ app: geospatial_data_converter
  spec:
  containers:
- - name: geospatial-data-converter
- image: joshuasundance/geospatial-data-converter:0.1.1
+ - name: geospatial_data_converter
+ image: joshuasundance/geospatial_data_converter:0.1.1
  imagePullPolicy: Always
  resources:
  requests:
@@ -29,15 +29,15 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
- name: geospatial-data-converter-service
+ name: geospatial_data_converter-service
  # configure on Azure and uncomment below to use a vnet
 # annotations:
 # service.beta.kubernetes.io/azure-load-balancer-internal: "true"
 # service.beta.kubernetes.io/azure-load-balancer-ipv4: vnet.ip.goes.here
-# service.beta.kubernetes.io/azure-dns-label-name: "geospatial-data-converter"
+# service.beta.kubernetes.io/azure-dns-label-name: "geospatial_data_converter"
 spec:
  selector:
- app: geospatial-data-converter
+ app: geospatial_data_converter
  ports:
  - protocol: TCP
  port: 80
@@ -47,11 +47,11 @@ spec:
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
- name: geospatial-data-converter-network-policy
+ name: geospatial_data_converter-network-policy
 spec:
  podSelector:
  matchLabels:
- app: geospatial-data-converter
+ app: geospatial_data_converter
  policyTypes:
  - Ingress
  ingress:

requirements.txt

@@ -3,5 +3,6 @@ beautifulsoup4==4.12.2
 geopandas==0.14.1
 lxml==4.9.3
 pyogrio==0.7.2
+pytest
 restgdf==0.9.7
 streamlit==1.28.2

tests/test_conversions.py

@@ -0,0 +1,26 @@
+import os
+import pytest
+from geospatial_data_converter.utils import convert, read_file
+
+input_exts = ["kml", "kmz", "geojson", "zip"]
+output_exts = ["KML", "ESRI Shapefile", "GeoJSON", "CSV", "OpenFileGDB"]
+output_format_dict = {
+ "ESRI Shapefile": "shp",
+ "GeoJSON": "geojson",
+ "CSV": "csv",
+ "KML": "kml",
+ "OpenFileGDB": "gdb",
+}
+
+
+@pytest.mark.parametrize("in_ext", input_exts)
+@pytest.mark.parametrize("out_ext", output_exts)
+def test_kml_coversion(in_ext: str, out_ext: str) -> None:
+ test_file = f"test.{in_ext}"
+ test_file_path = os.path.join(os.getcwd(), "tests", "test_data", test_file)
+ with open(test_file_path, "rb") as f:
+ kml = read_file(f)
+ out_file = f"test.{output_format_dict[out_ext]}"
+ converted_data = convert(kml, out_file, out_ext)
+ with open("test.kml", "wb") as f:
+ f.write(converted_data)