Update app.py

app.py

@@ -25,12 +25,12 @@ region_df = load_region_data(region_lookup_path)
 ####################  Create the embeddings collection and save ######################
 # the steps below need to be performed only once and then commented out any unnecssary compute over-run
 ##### First we process and create the chunks for relvant data source
-chunks = process_giz_worldwide()
+#chunks = process_giz_worldwide()
 ##### Convert to langchain documents
-temp_doc = create_documents(chunks,'chunks')
+#temp_doc = create_documents(chunks,'chunks')
 ##### Embed and store docs, check if collection exist then you need to update the collection
 collection_name = "giz_worldwide"
-hybrid_embed_chunks(docs=temp_doc, collection_name=collection_name, del_if_exists=True)
+#hybrid_embed_chunks(docs=temp_doc, collection_name=collection_name, del_if_exists=True)
 
 ################### Hybrid Search ######################################################
 client = get_client()
@@ -47,6 +47,7 @@ _, unique_sub_regions = get_regions(region_df)
 def get_country_name_and_region_mapping(_client, collection_name, region_df):
     results = hybrid_search(_client, "", collection_name)
     country_set = set()
+    
     for res in results[0] + results[1]:
         countries = res.payload.get('metadata', {}).get('countries', "[]")
         try:
@@ -94,23 +95,23 @@ else:
 with col2:
     country_filter = st.selectbox("Country", ["All/Not allocated"] + filtered_country_names)  # Display filtered country names
 
-# Year range slider
-with col3:
-    current_year = datetime.now().year
-    default_start_year = current_year - 5
+# # Year range slider # ToDo add end_year filter again
+# with col3:
+#     current_year = datetime.now().year
+#     default_start_year = current_year - 5
 
-    # 3) The max_value is now the actual max end_year from collection
-    end_year_range = st.slider(
-        "Project End Year",
-        min_value=2010,
-        max_value=max_end_year,
-        value=(default_start_year, max_end_year),
-    )
+#     # 3) The max_value is now the actual max end_year from collection
+#     end_year_range = st.slider(
+#         "Project End Year",
+#         min_value=2010,
+#         max_value=max_end_year,
+#         value=(default_start_year, max_end_year),
+#     )
 
 # Checkbox to control whether to show only exact matches
 show_exact_matches = st.checkbox("Show only exact matches", value=False)
 
-def filter_results(results, country_filter, region_filter, end_year_range):
+def filter_results(results, country_filter, region_filter): ## , end_year_range ToDo add end_year filter again
     filtered = []
     for r in results:
         metadata = r.payload.get('metadata', {})
@@ -145,7 +146,7 @@ def filter_results(results, country_filter, region_filter, end_year_range):
         if (
             (country_filter == "All/Not allocated" or selected_iso_code in c_list)
             and (region_filter == "All/Not allocated" or countries_in_region)
-            and (end_year_range[0] <= end_year_val <= end_year_range[1])
+#            and (end_year_range[0] <= end_year_val <= end_year_range[1]) # ToDo add end_year filter again
         ):
             filtered.append(r)
     return filtered
@@ -161,20 +162,20 @@ lexical_all = results[1]
 
 # 2) Filter out content < 20 chars (as intermediate fix to problem that e.g. super short paragraphs with few chars get high similarity score)
 semantic_all = [
-    r for r in semantic_all if len(r.payload["page_content"]) >= 20
+    r for r in semantic_all if len(r.payload["page_content"]) >= 5
 ]
 lexical_all = [
-    r for r in lexical_all if len(r.payload["page_content"]) >= 20
+    r for r in lexical_all if len(r.payload["page_content"]) >= 5
 ]
 
 # 2) Apply a threshold to SEMANTIC results (score >= 0.4)
-semantic_thresholded = [r for r in semantic_all if r.score >= 0.4]
+semantic_thresholded = [r for r in semantic_all if r.score >= 0.0]
 
 # 2) Filter the entire sets
-filtered_semantic = filter_results(semantic_thresholded, country_filter, region_filter, end_year_range)
-filtered_lexical = filter_results(lexical_all, country_filter, region_filter, end_year_range)
+filtered_semantic = filter_results(semantic_thresholded, country_filter, region_filter) ## , end_year_range ToDo add end_year filter again
+filtered_lexical = filter_results(lexical_all, country_filter, region_filter)## , end_year_range ToDo add end_year filter again
 
-filtered_semantic_no_dupe = remove_duplicates(filtered_semantic)
+filtered_semantic_no_dupe = remove_duplicates(filtered_semantic) # ToDo remove duplicates again?
 filtered_lexical_no_dupe = remove_duplicates(filtered_lexical)
 
 
@@ -197,8 +198,8 @@ if show_exact_matches:
 
     # 3) Now apply your region/country/year filter on that new list
     filtered_lexical = filter_results(
-        lexical_substring_filtered, country_filter, region_filter, end_year_range
-    )
+        lexical_substring_filtered, country_filter, region_filter
+    ) ## , end_year_range ToDo add end_year filter again
 
     # 4) Remove duplicates
     filtered_lexical_no_dupe = remove_duplicates(filtered_lexical)
@@ -216,7 +217,7 @@ if show_exact_matches:
             # Snippet logic (80 words)
             full_text = res.payload['page_content']
             words = full_text.split()
-            preview_word_count = 80
+            preview_word_count = 200
             preview_text = " ".join(words[:preview_word_count])
             remainder_text = " ".join(words[preview_word_count:])
             st.write(preview_text + ("..." if remainder_text else ""))
@@ -232,6 +233,10 @@ if show_exact_matches:
             client_name = metadata.get('client', 'Unknown Client')
             start_year = metadata.get('start_year', None)
             end_year = metadata.get('end_year', None)
+            total_volume = metadata.get('total_volume', "Unknown")
+            total_project = metadata.get('total_project', "Unknown")
+            id = metadata.get('id', "Unknown")
+
             
             try:
                 c_list = json.loads(countries.replace("'", '"'))
@@ -255,18 +260,16 @@ if show_exact_matches:
             start_year_str = f"{int(round(float(start_year)))}" if start_year else "Unknown"
             end_year_str = f"{int(round(float(end_year)))}" if end_year else "Unknown"
             
-            # Build the final string
             if matched_countries:
-                # We have at least 1 valid country name
                 additional_text = (
                     f"**{', '.join(matched_countries)}**, commissioned by **{client_name}**, "
-                    f"**{start_year_str}-{end_year_str}**"
+                    f"**{start_year_str}-{end_year_str}**, project ID: {id}, project budget: {total_project}, total volumne: {total_volume}"
                 )
             else:
-                # No valid countries found
                 additional_text = (
-                    f"Commissioned by **{client_name}**, **{start_year_str}-{end_year_str}**"
+                    f"Commissioned by **{client_name}**, **{start_year_str}-{end_year_str}**, project ID: {id}, project budget: {total_project}, total volumne: {total_volume}"
                 )
+
             
             st.markdown(additional_text)
             st.divider()
@@ -302,6 +305,9 @@ else:
             client_name = metadata.get('client', 'Unknown Client')
             start_year = metadata.get('start_year', None)
             end_year = metadata.get('end_year', None)
+            total_volume = metadata.get('total_volume', "Unknown")
+            total_project = metadata.get('total_project', "Unknown")
+            id = metadata.get('id', "Unknown")
             
             try:
                 c_list = json.loads(countries.replace("'", '"'))
@@ -327,16 +333,15 @@ else:
             
             # Build the final string
             if matched_countries:
-                # We have at least 1 valid country name
                 additional_text = (
                     f"**{', '.join(matched_countries)}**, commissioned by **{client_name}**, "
-                    f"**{start_year_str}-{end_year_str}**"
+                    f"**{start_year_str}-{end_year_str}**, project ID: {id}, project budget: {total_project}, total volumne: {total_volume}"
                 )
             else:
-                # No valid countries found
                 additional_text = (
-                    f"Commissioned by **{client_name}**, **{start_year_str}-{end_year_str}**"
+                    f"Commissioned by **{client_name}**, **{start_year_str}-{end_year_str}**, project ID: {id}, project budget: {total_project}, total volumne: {total_volume}"
                 )
+
             
             st.markdown(additional_text)
             st.divider()