feat: add streamlit frontend

.gitignore

@@ -1,6 +1,8 @@
 .aider*
 .envrc
 
+.streamlit/secrets.toml
+auth/
 chroma_themesdb/
 data/
 notes/

app.py

@@ -1,114 +1,158 @@
+import polars as pl
 import py7zr
 import streamlit as st
+import streamlit_authenticator as stauth
 
 from planning_ai.common.utils import Paths
 from planning_ai.main import main as report_main
 from planning_ai.preprocessing.azure_doc import azure_process_pdfs
 from planning_ai.preprocessing.gcpt3 import main as preprocess_main
 
+auth = st.secrets.to_dict()
+
+authenticator = stauth.Authenticate(
+    auth["credentials"],
+    auth["cookie"]["name"],
+    auth["cookie"]["key"],
+    auth["cookie"]["expiry_days"],
+)
+
 UPLOAD_DIR = Paths.RAW / "gcpt3"
 
+try:
+    authenticator.login()
+except Exception as e:
+    st.error(e)
+
 if "files_extracted" not in st.session_state:
     st.session_state["files_extracted"] = False
+if "completed" not in st.session_state:
+    st.session_state["completed"] = False
 
-st.title("Report Builder")
-
+if st.session_state["authentication_status"]:
+    authenticator.logout()
+    st.write("---")
 
-st.header("Upload JDL response `.json` files")
-st.write(
-    "Upload your `.json` files here as a `7zip` file, they will be saved to the `data/raw/gcpt3` directory."
-)
+    st.title("Report Builder")
 
-with st.expander("File Format"):
+    st.header("Upload JDL response `.json` files")
     st.write(
-        """
-    The `.json` files should look like the following:
-
-    ```json
-    {
-        "id": 10008,
-        "method": "Paper",
-        "respondentpostcode": "CB2 9NE",
-        "text": "",
-        "attachments": [
-            {
-                "id": 3803,
-                "url": "http:\/\/www.cambridge.gov.uk\/public\/ldf\/localplan2031\/15417.pdf",
-                "published": false
-            }
-        ],
-        "representations": [
-            {
-                "id": 15417,
-                "support\/object": "Object",
-                "document": "Issues and Options Report",
-                "documentelementid": 29785,
-                "documentelementtitle": "3 - Spatial Strategy, Question 3.10",
-                "summary": "No more green belt taken away, which is prime agricultural land. Noise pollution & light pollution for surrounding villages and new houses being built, no bus services either!"
-            },
-        ]
-    }
-    ```
-"""
+        "Upload your `.json` files here as a `7zip` file, they will be saved to the `data/raw/gcpt3` directory."
     )
-if uploaded_file := st.file_uploader("Choose a `.7z` file:", type="7z"):
-    with st.spinner("Extracting files..."):
-        try:
-            with py7zr.SevenZipFile(uploaded_file, mode="r") as archive:
-                archive.extractall(path=UPLOAD_DIR)
-            st.session_state["files_extracted"] = True
-            st.success(
-                f"Extracted `{len(list(UPLOAD_DIR.glob('*.json')))}` files to `{UPLOAD_DIR}`."
-            )
-        except Exception as e:
-            st.error(f"Failed to extract files {e}")
-
-if not st.session_state["files_extracted"]:
-    st.write("No files uploaded yet.")
-
-st.write("---")
-
-if st.session_state["files_extracted"]:
-    st.title("Build Report")
-    st.write(
-        "Once the files are extracted, click the button below to build the report."
-    )
-    if st.button("Build Report", type="primary"):
-        with st.spinner("Preprocessing files..."):
-            try:
-                preprocess_main()
-                st.success("Preprocessing completed successfully!")
-            except Exception as e:
-                st.error(f"An error occurred during preprocessing: {e}")
-        with st.spinner("Extracting text from PDFs..."):
+
+    with st.expander("File Format"):
+        st.write(
+            """
+        The `.json` files should look like the following:
+
+        ```json
+        {
+            "id": 10008,
+            "method": "Paper",
+            "respondentpostcode": "CB2 9NE",
+            "text": "",
+            "attachments": [
+                {
+                    "id": 3803,
+                    "url": "http:\/\/www.cambridge.gov.uk\/public\/ldf\/localplan2031\/15417.pdf",
+                    "published": false
+                }
+            ],
+            "representations": [
+                {
+                    "id": 15417,
+                    "support\/object": "Object",
+                    "document": "Issues and Options Report",
+                    "documentelementid": 29785,
+                    "documentelementtitle": "3 - Spatial Strategy, Question 3.10",
+                    "summary": "No more green belt taken away, which is prime agricultural land. Noise pollution & light pollution for surrounding villages and new houses being built, no bus services either!"
+                },
+            ]
+        }
+        ```
+    """
+        )
+    if uploaded_file := st.file_uploader("Choose a `.7z` file:", type="7z"):
+        with st.spinner("Extracting files..."):
             try:
-                azure_process_pdfs()
-                st.success("Text extraction completed successfully!")
-            except Exception as e:
-                st.error(f"An error occurred during PDF text extraction: {e}")
-        with st.spinner("Building report..."):
-            report_main()
-            report_path = Paths.SUMMARY / "Summary_Documents.pdf"
-            summaries_path = Paths.SUMMARY / "Summary_of_Submitted_Responses.pdf"
+                # remove old files
+                _ = [file.unlink() for file in UPLOAD_DIR.glob("*.json")]
 
-            if report_path.exists() and summaries_path.exists():
+                with py7zr.SevenZipFile(uploaded_file, mode="r") as archive:
+                    archive.extractall(path=UPLOAD_DIR)
+                st.session_state["files_extracted"] = True
                 st.success(
-                    "Report built successfully! Please click download buttons below."
+                    f"Extracted `{len(list(UPLOAD_DIR.glob('*.json')))}` files to `{UPLOAD_DIR}`."
+                )
+            except Exception as e:
+                st.error(f"Failed to extract files {e}")
+
+    if not st.session_state["files_extracted"]:
+        st.write("No files uploaded yet.")
+
+    st.write("---")
+
+    if st.session_state["files_extracted"] and not st.session_state["completed"]:
+        st.title("Build Report")
+        st.write(
+            "Once the files are extracted, click the button below to build the report."
+        )
+        if st.button("Build Report", type="primary"):
+            with st.spinner("Preprocessing files..."):
+                try:
+                    preprocess_main()
+                    st.success("Preprocessing completed successfully!")
+                except Exception as e:
+                    st.error(f"An error occurred during preprocessing: {e}")
+            with st.spinner("Extracting text from PDFs..."):
+                try:
+                    azure_process_pdfs()
+                    st.success("Text extraction completed successfully!")
+                except Exception as e:
+                    st.error(f"An error occurred during PDF text extraction: {e}")
+            with st.spinner("Building report..."):
+                representations_documents = report_main()
+                st.session_state["completed"] = True
+elif st.session_state["authentication_status"] is False:
+    st.error("Username/password is incorrect")
+elif st.session_state["authentication_status"] is None:
+    st.warning("Please enter your username and password")
+
+if st.session_state["completed"]:
+    representations_documents = (
+        pl.read_parquet(Paths.STAGING / "gcpt3.parquet")["representations_document"]
+        .unique()
+        .to_list()
+    )
+    representations_documents_dash = [
+        "-".join(rep.split(" ")) for rep in representations_documents
+    ]
+
+    st.success("Reports built successfully! Please click download buttons below.")
+    for rep_dash, rep in zip(representations_documents_dash, representations_documents):
+        report_path = Paths.SUMMARY / f"Summary_Documents-{rep_dash}.pdf"
+        summaries_path = (
+            Paths.SUMMARY / f"Summary_of_Submitted_Responses-{rep_dash}.pdf"
+        )
+
+        col1, col2 = st.columns(2, border=True)
+        with col1:
+            with open(summaries_path, "rb") as pdf_file:
+                st.markdown("**Representations Summary Download**")
+                st.download_button(
+                    label=f"{rep}",
+                    data=pdf_file,
+                    file_name=f"Summary_of_Submitted_Responses-{rep_dash}.pdf",
+                    mime="application/pdf",
+                    type="primary",
+                )
+        with col2:
+            with open(report_path, "rb") as pdf_file:
+                st.markdown("**Executive Report Download**")
+                st.download_button(
+                    label=f"{rep}",
+                    data=pdf_file,
+                    file_name=f"Summary_Documents-{rep_dash}.pdf",
+                    mime="application/pdf",
+                    type="primary",
                 )
-                col1, col2 = st.columns(2)
-                with col1:
-                    with open(summaries_path, "rb") as docx_file:
-                        st.download_button(
-                            label="Download Report",
-                            data=docx_file,
-                            file_name="Summary_of_Submitted_Responses.docx",
-                            mime="application/docx",
-                        )
-                with col2:
-                    with open(report_path, "rb") as docx_file:
-                        st.download_button(
-                            label="Download Summaries Report",
-                            data=docx_file,
-                            file_name="Summary_Documents.docx",
-                            mime="application/docx",
-                        )

pyproject.toml

@@ -27,7 +27,6 @@ dependencies = [
     "pypdf2>=3.0.1",
     "beautifulsoup4>=4.12.3",
     "chromadb>=0.5.23",
-    # "langchain-chroma>=0.1.4",
     "pypdf>=5.1.0",
     "geopandas>=1.0.1",
     "azure-ai-documentintelligence>=1.0.0",
@@ -36,6 +35,9 @@ dependencies = [
     "mapclassify>=2.8.1",
     "streamlit>=1.41.1",
     "py7zr>=0.22.0",
+    "streamlit-authenticator>=0.4.1",
+    "en_core_web_lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl"
+
 ]
 
 [tool.uv]

uv.lock

@@ -322,6 +322,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/ec/4e/de4ff18bcf55857ba18d3a4bd48c8a9fde6bb0980c9d20b263f05387fd88/cachetools-5.5.1-py3-none-any.whl", hash = "sha256:b76651fdc3b24ead3c648bbdeeb940c1b04d365b38b4af66788f9ec4a81d42bb", size = 9530 },
 ]
 
+[[package]]
+name = "captcha"
+version = "0.6.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pillow" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/48/4c/547742298d99408a7172aa5b1328b6bb9d85234b4082e0fd379f9b893615/captcha-0.6.0.tar.gz", hash = "sha256:a5ead7679cd8477bc636524d54f50e092a4b6f3db4e52bb98a689593638d1772", size = 187610 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c0/f2/9af6abd7b07e4709a7d698fee6fa7ce7fb9814886b18de61580311b0fe96/captcha-0.6.0-py3-none-any.whl", hash = "sha256:148b2475be00a067b6173b7796cdc2332dbcb6fd4bfced1055bc8f69846bc924", size = 102241 },
+]
+
 [[package]]
 name = "catalogue"
 version = "2.0.10"
@@ -635,6 +647,14 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/4c/a3/ac312faeceffd2d8f86bc6dcb5c401188ba5a01bc88e69bed97578a0dfcd/durationpy-0.9-py3-none-any.whl", hash = "sha256:e65359a7af5cedad07fb77a2dd3f390f8eb0b74cb845589fa6c057086834dd38", size = 3461 },
 ]
 
+[[package]]
+name = "en-core-web-lg"
+version = "3.8.0"
+source = { url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl" }
+wheels = [
+    { url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl", hash = "sha256:293e9547a655b25499198ab15a525b05b9407a75f10255e405e8c3854329ab63" },
+]
+
 [[package]]
 name = "eval-type-backport"
 version = "0.2.2"
@@ -662,6 +682,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7b/8f/c4d9bafc34ad7ad5d8dc16dd1347ee0e507a52c3adb6bfa8887e1c6a26ba/executing-2.2.0-py2.py3-none-any.whl", hash = "sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa", size = 26702 },
 ]
 
+[[package]]
+name = "extra-streamlit-components"
+version = "0.1.71"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "streamlit" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/16/a7/580b13af828ef38888196f8b2c03fa97afa89cdb7946438ca5f3271e9a81/extra_streamlit_components-0.1.71.tar.gz", hash = "sha256:d18314cf2ed009f95641882b50aa3bdb11b6a0eb6403fb43dbc8af1722419617", size = 2250093 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/25/57/1115e9b974478fac83ba9cd79def8b3770a91b7a9001c46a76491071f2fe/extra_streamlit_components-0.1.71-py3-none-any.whl", hash = "sha256:c8e6f98446adecd3002756362e50d0669693b7673afaa89cebfced6415cc6bd3", size = 4858597 },
+]
+
 [[package]]
 name = "fastapi"
 version = "0.115.8"
@@ -2044,6 +2076,7 @@ dependencies = [
     { name = "azure-ai-documentintelligence" },
     { name = "beautifulsoup4" },
     { name = "chromadb" },
+    { name = "en-core-web-lg" },
     { name = "fastexcel" },
     { name = "geopandas" },
     { name = "langchain-community" },
@@ -2069,6 +2102,7 @@ dependencies = [
     { name = "spacy" },
     { name = "spacytextblob" },
     { name = "streamlit" },
+    { name = "streamlit-authenticator" },
     { name = "tabulate" },
     { name = "transformers" },
 ]
@@ -2084,6 +2118,7 @@ requires-dist = [
     { name = "azure-ai-documentintelligence", specifier = ">=1.0.0" },
     { name = "beautifulsoup4", specifier = ">=4.12.3" },
     { name = "chromadb", specifier = ">=0.5.23" },
+    { name = "en-core-web-lg", url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl" },
     { name = "fastexcel", specifier = ">=0.11.6" },
     { name = "geopandas", specifier = ">=1.0.1" },
     { name = "langchain-community", specifier = ">=0.2.16" },
@@ -2109,6 +2144,7 @@ requires-dist = [
     { name = "spacy", specifier = ">=3.7.6" },
     { name = "spacytextblob", specifier = ">=4.0.0" },
     { name = "streamlit", specifier = ">=1.41.1" },
+    { name = "streamlit-authenticator", specifier = ">=0.4.1" },
     { name = "tabulate", specifier = ">=0.9.0" },
     { name = "transformers", specifier = ">=4.44.2" },
 ]
@@ -2485,6 +2521,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293 },
 ]
 
+[[package]]
+name = "pyjwt"
+version = "2.10.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e7/46/bd74733ff231675599650d3e47f361794b22ef3e3770998dda30d3b63726/pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953", size = 87785 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997 },
+]
+
 [[package]]
 name = "pyogrio"
 version = "0.10.0"
@@ -3171,6 +3216,23 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c2/87/b2e162869500062a94dde7589c167367b5538dab6eacce2e7c0f00d5c9c5/streamlit-1.41.1-py2.py3-none-any.whl", hash = "sha256:0def00822480071d642e6df36cd63c089f991da3a69fd9eb4ab8f65ce27de4e0", size = 9100386 },
 ]
 
+[[package]]
+name = "streamlit-authenticator"
+version = "0.4.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "bcrypt" },
+    { name = "captcha" },
+    { name = "extra-streamlit-components" },
+    { name = "pyjwt" },
+    { name = "pyyaml" },
+    { name = "streamlit" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/f8/70/04c24fb2d9b372edfa2261cbec3828633bc997c4bf29310ff53cf092a647/streamlit_authenticator-0.4.1.tar.gz", hash = "sha256:d4afeec5c972d794052a5552aa570e7eb620f0cc183484a5b052304907268381", size = 31401 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/93/28/c19eae3f2b29dd4658e16d3fd3e0164af13c501493e0bc260effcfe04407/streamlit_authenticator-0.4.1-py3-none-any.whl", hash = "sha256:776136e6a6ecde1ecc9dda054a7492290199b81e19268abff307a7d508685942", size = 34747 },
+]
+
 [[package]]
 name = "sympy"
 version = "1.13.3"