Update app.py

app.py

@@ -7,7 +7,11 @@ import configparser
 from datetime import datetime
 from torch import cuda
 
-# Import existing modules from appStore
+# ------------------------------------------------------------------------------
+# Import modules from the appStore package
+# These modules handle data preparation, embedding, search, region handling,
+# retrieval of RAG answers, and filtering utilities.
+# ------------------------------------------------------------------------------
 from appStore.prep_data import process_giz_worldwide, remove_duplicates, get_max_end_year, extract_year
 from appStore.prep_utils import create_documents, get_client
 from appStore.embed import hybrid_embed_chunks
@@ -19,10 +23,9 @@ from appStore.region_utils import (
     get_regions, 
     get_country_name_and_region_mapping  
 )
-# TF-IDF part (excluded from the app for now)
+# Note: The TF-IDF extraction is currently not used in the app.
 # from appStore.tfidf_extraction import extract_top_keywords
 
-# Import helper modules, including format_project_id for formatting IDs
 from appStore.rag_utils import (
     highlight_query,
     get_rag_answer,
@@ -34,52 +37,57 @@ from appStore.filter_utils import (
     filter_results,
     get_crs_options
 )
-
 from appStore.crs_utils import lookup_crs_value
 
-###########################################
-# Model Config
-###########################################
+# ------------------------------------------------------------------------------
+# Model Configuration
+# ------------------------------------------------------------------------------
+# Read model parameters from configuration file
 config = configparser.ConfigParser()
 config.read('model_params.cfg')
-
 DEDICATED_MODEL = config.get('MODEL', 'DEDICATED_MODEL')
 DEDICATED_ENDPOINT = config.get('MODEL', 'DEDICATED_ENDPOINT')
 WRITE_ACCESS_TOKEN = st.secrets["Llama_3_1"]
 
+# Set page configuration for Streamlit
 st.set_page_config(page_title="SEARCH IATI", layout='wide')
 
-###########################################
-# Cache the project data
-###########################################
+# ------------------------------------------------------------------------------
+# Load and Cache Project Data
+# ------------------------------------------------------------------------------
 @st.cache_data
 def load_project_data():
     """
-    Load and process the GIZ worldwide data, returning a pandas DataFrame.
+    Load and process the GIZ worldwide project data.
+    
+    Returns:
+        pd.DataFrame: Processed project data as a pandas DataFrame.
     """
     return process_giz_worldwide()
 
 project_data = load_project_data()
 
-# Determine min and max budgets in million euros
+# ------------------------------------------------------------------------------
+# Calculate Budget Range (in million euros)
+# ------------------------------------------------------------------------------
 budget_series = pd.to_numeric(project_data['total_project'], errors='coerce').dropna()
 min_budget_val = float(budget_series.min() / 1e6)
 max_budget_val = float(budget_series.max() / 1e6)
 
-###########################################
-# Prepare region data
-###########################################
+# ------------------------------------------------------------------------------
+# Prepare Region Data
+# ------------------------------------------------------------------------------
 region_lookup_path = "docStore/regions_lookup.csv"
 region_df = load_region_data(region_lookup_path)
 
-###########################################
-# Get device
-###########################################
+# ------------------------------------------------------------------------------
+# Determine Device for Computation
+# ------------------------------------------------------------------------------
 device = 'cuda' if cuda.is_available() else 'cpu'
 
-###########################################
-# Streamlit App Layout
-###########################################
+# ------------------------------------------------------------------------------
+# Layout: Header and About Section
+# ------------------------------------------------------------------------------
 col_title, col_about = st.columns([8, 2])
 with col_title:
     st.markdown("<h1 style='text-align:center;'>GIZ Project Search (PROTOTYPE)</h1>", unsafe_allow_html=True)
@@ -87,30 +95,32 @@ with col_about:
     with st.expander("ℹ️ About"):
         st.markdown(
             """
-            This app is a prototype for testing purposes using publicly available project data 
-            from the German International Cooperation Society (GIZ) as of 23rd February 2025.  
+            This prototype app uses publicly available project data from the German 
+            International Cooperation Society (GIZ) as of 23rd February 2025.  
             **Please do NOT enter sensitive or personal information.**  
-            **Note**: The answers are AI-generated and may be wrong or misleading.
+            **Note**: The answers are AI-generated and may be incorrect or misleading.
             """, unsafe_allow_html=True
         )
 
-
-###########################################
-# Create or load the embeddings collection
-###########################################
+# ------------------------------------------------------------------------------
+# Create or Load the Embeddings Collection
+# ------------------------------------------------------------------------------
 collection_name = "giz_worldwide"
 client = get_client()
+
+# Display existing collections for debugging purposes
 print(client.get_collections())
 
-# Uncomment if needed:
+# Uncomment the block below if you need to reprocess and embed documents.
 # chunks = process_giz_worldwide()
 # temp_doc = create_documents(chunks, 'chunks')
 # hybrid_embed_chunks(docs=temp_doc, collection_name=collection_name, del_if_exists=True)
 
+# Retrieve maximum project end year and region mapping
 max_end_year = get_max_end_year(client, collection_name)
 _, unique_sub_regions = get_regions(region_df)
 
-# Build country->code and code->region mapping
+# Build mapping between country names and region codes
 country_name_mapping, iso_code_to_sub_region = get_country_name_and_region_mapping(
     client, 
     collection_name, 
@@ -121,10 +131,13 @@ country_name_mapping, iso_code_to_sub_region = get_country_name_and_region_mappi
 )
 unique_country_names = sorted(country_name_mapping.keys())
 
-###########################################
-# Define reset_filters function using session_state
-###########################################
+# ------------------------------------------------------------------------------
+# Session State Reset Functions
+# ------------------------------------------------------------------------------
 def reset_filters():
+    """
+    Reset all filter options in the session state to their default values.
+    """
     st.session_state["region_filter"] = "All/Not allocated"
     st.session_state["country_filter"] = "All/Not allocated"
     current_year = datetime.now().year
@@ -138,19 +151,24 @@ def reset_filters():
     st.session_state["page"] = 1
 
 def reset_page():
+    """
+    Reset the pagination page to the first page.
+    """
     st.session_state.page = 1
 
-###########################################
-# Main query input
-###########################################
-var = st.text_input("Enter Question", key="query",  on_change=reset_page)
-###########################################
-# Filter Controls - Row 1
-###########################################
-col1, col2, col3, col4, col5 = st.columns([1, 1, 1, 1, 1])
+# ------------------------------------------------------------------------------
+# Main Query Input
+# ------------------------------------------------------------------------------
+var = st.text_input("Enter Question", key="query", on_change=reset_page)
 
+# ------------------------------------------------------------------------------
+# Filter Controls - Row 1: Basic Filters
+# ------------------------------------------------------------------------------
+col1, col2, col3, col4, col5 = st.columns([1, 1, 1, 1, 1])
 with col1:
-    region_filter = st.selectbox("Region", ["All/Not allocated"] + sorted(unique_sub_regions), key="region_filter", on_change=reset_page)
+    region_filter = st.selectbox("Region", ["All/Not allocated"] + sorted(unique_sub_regions),
+                                 key="region_filter", on_change=reset_page)
+# If a specific region is selected, filter the country names accordingly.
 if region_filter == "All/Not allocated":
     filtered_country_names = unique_country_names
 else:
@@ -160,7 +178,8 @@ else:
     ]
 
 with col2:
-    country_filter = st.selectbox("Country", ["All/Not allocated"] + filtered_country_names, key="country_filter", on_change=reset_page)
+    country_filter = st.selectbox("Country", ["All/Not allocated"] + filtered_country_names,
+                                  key="country_filter", on_change=reset_page)
 
 with col3:
     current_year = datetime.now().year
@@ -188,14 +207,14 @@ with col5:
         on_change=reset_page
     )
 
-###########################################
-# Filter Controls - Row 2 (Additional Filters)
-###########################################
+# ------------------------------------------------------------------------------
+# Filter Controls - Row 2: Additional Filters
+# ------------------------------------------------------------------------------
 col1_2, col2_2, col3_2, col4_2, col5_2 = st.columns(5)
-
 with col1_2:
     client_options = sorted(project_data["client"].dropna().unique().tolist())
     client_filter = st.selectbox("Client", ["All/Not allocated"] + client_options, key="client_filter")
+# Columns 2 to 5 are left empty for layout alignment
 with col2_2:
     st.empty()
 with col3_2:
@@ -205,51 +224,59 @@ with col4_2:
 with col5_2:
     st.empty()
 
-
-###########################################
-# Filter Controls - Row 3 (Remaining Filter)
-###########################################
+# ------------------------------------------------------------------------------
+# Filter Controls - Row 3: Toggle and Reset Button
+# ------------------------------------------------------------------------------
 col_left, col_right = st.columns([11, 1])
 with col_left:
-    # Place the "Show only exact matches" checkbox here (or any other widgets)
+    # Checkbox to toggle exact match filtering
     show_exact_matches = st.checkbox("Show only exact matches", key="show_exact_matches", on_change=reset_page)
 with col_right:
-    # The reset button will now be in a narrow column on the far right.
+    # Reset filters button (right-aligned)
     with st.container():
         st.markdown("<div style='text-align: right;'>", unsafe_allow_html=True)
         if st.button("**Reset Filters**", key="reset_button_row3"):
             reset_filters()
         st.markdown("</div>", unsafe_allow_html=True)
 
-
-###########################################
-# Helper function for valid project id
-###########################################
+# ------------------------------------------------------------------------------
+# Helper Function: Validate Project ID
+# ------------------------------------------------------------------------------
 def valid_project_id(pid_str):
+    """
+    Check if the provided project ID string is valid.
+
+    Args:
+        pid_str (str): The project ID string.
+
+    Returns:
+        bool: True if the project ID is valid, False otherwise.
+    """
     if not pid_str:
         return False
     if pid_str.lower() in ["nan", "none"]:
         return False
     return True
 
-###########################################
-# Main Search / Results
-###########################################
+# ------------------------------------------------------------------------------
+# Main Search and Display Logic
+# ------------------------------------------------------------------------------
 if not var.strip():
+    # Inform the user if no query is entered.
     st.info("Please enter a question to see results.")
 else:
-    # 1) Perform hybrid search
+    # --- 1. Execute Hybrid Search ---
     results = hybrid_search(client, var, collection_name, limit=500)
     semantic_all, lexical_all = results[0], results[1]
 
-    # Filter out short pages
+    # Filter out results with very short page content
     semantic_all = [r for r in semantic_all if len(r.payload["page_content"]) >= 5]
     lexical_all = [r for r in lexical_all if len(r.payload["page_content"]) >= 5]
 
-    # Apply threshold to semantic results if desired
+    # Apply a threshold to semantic search scores if needed
     semantic_thresholded = [r for r in semantic_all if r.score >= 0.25]
 
-    # 2) Filter results based on the user’s selections
+    # --- 2. Apply User-Selected Filters ---
     filtered_semantic = filter_results(
         semantic_thresholded, 
         country_filter, 
@@ -275,29 +302,38 @@ else:
         get_country_name
     )
 
-    # Additional filter by client
+    # Additional filtering by client if selected
     if client_filter != "All/Not allocated":
         filtered_semantic = [r for r in filtered_semantic if r.payload.get("metadata", {}).get("client", "Unknown Client") == client_filter]
         filtered_lexical = [r for r in filtered_lexical if r.payload.get("metadata", {}).get("client", "Unknown Client") == client_filter]
 
-    # Remove duplicates
+    # Remove duplicate entries from the results
     filtered_semantic_no_dupe = remove_duplicates(filtered_semantic)
     filtered_lexical_no_dupe = remove_duplicates(filtered_lexical)
 
     def format_currency(value):
+        """
+        Format a numerical value as a currency string in euros.
+
+        Args:
+            value: The value to format.
+
+        Returns:
+            str: Formatted currency string.
+        """
         try:
             return f"€{int(float(value)):,}"
         except (ValueError, TypeError):
             return value
 
-    # --- Reprint Query (Left Aligned with "Query:") ---
+    # --- Reprint the user query for clarity ---
     st.markdown(
         f"<div style='text-align: left; font-size:2.1em; font-style: italic; font-weight: bold;'>Query: {var}</div>",
         unsafe_allow_html=True
     )
 
-    # 3) Display results
-    # --- Lexical Search Results Branch ---
+    # --- 3. Display Search Results Based on Matching Mode ---
+    # Lexical (Exact Match) Search Results Branch
     if show_exact_matches:
         query_substring = var.strip().lower()
         lexical_substring_filtered = [
@@ -308,21 +344,19 @@ else:
         if not filtered_lexical_no_dupe:
             st.write('No exact matches, consider unchecking "Show only exact matches"')
         else:
-            top_results = filtered_lexical_no_dupe  # Show all matching lexical results
-    
-            # --- Pagination (Above Lexical Results) in one row ---
+            top_results = filtered_lexical_no_dupe  # Use all matching lexical results
+
+            # --- Pagination Setup ---
             page_size = 15
             total_results = len(top_results)
             total_pages = (total_results - 1) // page_size + 1
             if "page" not in st.session_state:
                 st.session_state.page = 1
             current_page = st.session_state.page
-    
-            # Format the page numbers (highlight current page if not 1)
+
+            # Display current page info
             page_num = f"<b style='color: green;'>{current_page}</b>" if current_page != 1 else f"<b>{current_page}</b>"
             total_pages_str = f"<b>{total_pages}</b>"
-    
-            # Create two columns: one for the title and one for the selectbox
             col_title, col_pag = st.columns([13, 1])
             with col_title:
                 st.markdown(
@@ -333,20 +367,22 @@ else:
                 new_page_top = st.selectbox("Select Page", list(range(1, total_pages + 1)),
                                             index=current_page - 1, key="page_top")
                 st.session_state.page = new_page_top
-    
+
             start_index = (st.session_state.page - 1) * page_size
             end_index = start_index + page_size
             paged_results = top_results[start_index:end_index]
 
+            # Display each result with formatted metadata and content preview
             for i, res in enumerate(paged_results, start=start_index+1):
                 metadata = res.payload.get('metadata', {})
                 if "title" not in metadata:
                     metadata["title"] = compute_title(metadata)
+                # Highlight query text in the title
                 title_html = highlight_query(metadata["title"], var) if var.strip() else metadata["title"]
                 title_clean = re.sub(r'<a.*?>|</a>', '', title_html)
-                # Prepend the result number
                 st.markdown(f"#### {i}. **{title_clean}**", unsafe_allow_html=True)
 
+                # Prepare a description preview with an expandable "Show more" option
                 objective = metadata.get("objective", "None")
                 desc_en = metadata.get("description.en", "").strip()
                 desc_de = metadata.get("description.de", "").strip()
@@ -377,7 +413,8 @@ else:
                     new_crs_value = lookup_crs_value(crs_key_clean)
                     new_crs_value_clean = re.sub(r'\.0$', '', str(new_crs_value))
                     crs_combined = f"{crs_key_clean}: {new_crs_value_clean}" if crs_key_clean else "Unknown"
-                    
+
+                    # Process predecessor and successor project IDs if available
                     predecessor = metadata.get("predecessor_id", "").strip()
                     successor = metadata.get("successor_id", "").strip()
                     parts = []
@@ -394,8 +431,8 @@ else:
                             formatted_succ = successor
                         parts.append(f"**Successor Project:** {formatted_succ}")
                     extra_line = " | ".join(parts) if parts else ""
-                    
-                    # Build additional text with no extra blank row between Budget and predecessor/successor info
+
+                    # Build additional project information text
                     additional_text = (
                         f"**Objective:** {highlight_query(objective, var)}<br>"
                         f"**Commissioned by:** {metadata.get('client', 'Unknown Client')}<br>"
@@ -406,15 +443,17 @@ else:
                         additional_text += f"<br>{extra_line}"
                     additional_text += f"<br>**Country:** {country_raw}<br>**Sector:** {crs_combined}"
                     
+                    # Hide sensitive contact info if present
                     contact = metadata.get("contact", "").strip()
                     if contact and contact.lower() != "[email protected]":
                         additional_text += f"<br>**Contact:** [email protected]"
                     st.markdown(additional_text, unsafe_allow_html=True)
                 st.divider()
 
-            # Bottom pagination widget
+            # Bottom pagination widget for lexical results
             col_pag_bot = st.columns([11, 1])[1]
-            new_page_bot = col_pag_bot.selectbox("Select Page", list(range(1, total_pages + 1)), index=st.session_state.page - 1, key="page_bot")
+            new_page_bot = col_pag_bot.selectbox("Select Page", list(range(1, total_pages + 1)),
+                                                 index=st.session_state.page - 1, key="page_bot")
             st.session_state.page = new_page_bot
 
     # Semantic Search Results Branch
@@ -433,17 +472,15 @@ else:
             start_index = (st.session_state.page - 1) * page_size
             end_index = start_index + page_size
             top_results = filtered_semantic_no_dupe[start_index:end_index]
-            # --- RAG Answer (Left aligned, fully bold, highlighted in grey) ---
+            
+            # --- Retrieve and Format RAG Answer ---
             rag_answer = get_rag_answer(var, top_results, DEDICATED_ENDPOINT, WRITE_ACCESS_TOKEN)
             bullet_lines = []
             for line in rag_answer.splitlines():
                 if line.strip():
-                    # Remove any markdown bullet markers (e.g. "- " or "* ") from the start of the line
+                    # Clean and format the RAG answer lines
                     line = re.sub(r'^[-*]\s+', '', line.strip())
-                    # Convert markdown bold (e.g. **Title [2018.2101.6]**) to HTML <b> tags
                     line = re.sub(r'\*\*(.*?)\*\*', r'<b>\1</b>', line)
-                    # Optionally, bold any standalone numbers (if desired)
-                    # line = re.sub(r'(\d+)', r'<b>\1</b>', line)
                     bullet_lines.append(f"<li>{line}</li>")
             formatted_rag_answer = (
                 "<div style='background-color: #f0f0f0; padding: 10px;'>"
@@ -454,10 +491,9 @@ else:
             st.markdown(formatted_rag_answer, unsafe_allow_html=True)
             
             st.divider()
-            # Use two columns: one for the page info and one for the selectbox.
+            # Pagination controls for semantic results
             col_title, col_pag = st.columns([13, 1])
             with col_title:
-                # Format page numbers with bold formatting (green if not the first page)
                 page_num = f"<b style='color: green;'>{current_page}</b>" if current_page != 1 else f"<b>{current_page}</b>"
                 total_pages_str = f"<b>{total_pages}</b>"
                 st.markdown(
@@ -465,9 +501,11 @@ else:
                     unsafe_allow_html=True
                 )
             with col_pag:
-                new_page_top = st.selectbox("Select Page", list(range(1, total_pages + 1)), index=current_page - 1, key="page_top_sem")
+                new_page_top = st.selectbox("Select Page", list(range(1, total_pages + 1)),
+                                            index=current_page - 1, key="page_top_sem")
                 st.session_state.page = new_page_top
 
+            # Display each semantic result with detailed metadata and preview
             for i, res in enumerate(top_results, start=start_index+1):
                 metadata = res.payload.get('metadata', {})
                 if "title" not in metadata:
@@ -539,7 +577,8 @@ else:
                     st.markdown(additional_text, unsafe_allow_html=True)
                 st.divider()
 
-            # Bottom pagination widget (right aligned, 1/7 width)
+            # Bottom pagination widget for semantic results
             col_pag_bot = st.columns([13, 1])[1]
-            new_page_bot = col_pag_bot.selectbox("Select Page", list(range(1, total_pages + 1)), index=st.session_state.page - 1, key="page_bot_sem")
-            st.session_state.page = new_page_bot
+            new_page_bot = col_pag_bot.selectbox("Select Page", list(range(1, total_pages + 1)),
+                                                 index=st.session_state.page - 1, key="page_bot_sem")
+            st.session_state.page = new_page_bot