Update appStore/prep_data.py

appStore/prep_data.py

@@ -32,12 +32,53 @@ def process_iati():
 
     return projects_df
 
+
+# def process_giz_worldwide():
+#     """
+#     This will read the new giz_worldwide file and create the chunks.
+#     The following adjustments have been made:
+#       - Reads the file 'giz_worldwide_api_download_23_02_2025.json'
+#       - Renames 'name.en' to 'project_name'
+#       - Uses the 'merged_text' column for creating chunks and computing text size
+#       - Creates an empty 'url' column (since the new dataset has an empty URL)
+#       - Renames 'country' to 'countries'
+#       - Renames 'duration.project.start' to 'start_year' and 'duration.project.end' to 'end_year'
+#     """
+#     # Read the new JSON file
+#     giz_df = pd.read_json(f'{path_to_data}giz_worldwide/giz_worldwide_api_download_23_02_2025.json')
+    
+#     # Rename columns per new dataset requirements
+#     giz_df = giz_df.rename(columns={
+#         'name.en': 'project_name',
+#         'country': 'countries',
+#         'duration.project.start': 'start_year',
+#         'duration.project.end': 'end_year'
+#     })
+    
+#     # Create an empty 'url' column as the new dataset has an empty URL
+#     giz_df['url'] = ''
+    
+#     # Create text_size based on merged_text and create chunks from merged_text
+#     giz_df['text_size'] = giz_df['merged_text'].apply(lambda text: len(text.split()) if isinstance(text, str) else 0)
+#     giz_df['chunks'] = giz_df['merged_text'].apply(lambda text: create_chunks(text) if isinstance(text, str) else [])
+    
+#     print("initial df length:", len(giz_df))
+#     giz_df = giz_df.explode(column=['chunks'], ignore_index=True)
+#     print("new df length:", len(giz_df))
+#     print(giz_df.columns)
+    
+#     giz_df['source'] = 'GIZ_WORLDWIDE'
+#     return giz_df
+
+
 def process_giz_worldwide():
     """
     this will read the giz_worldwide files and create the chunks
     """
     giz_df = pd.read_json(f'{path_to_data}giz_worldwide/data_giz_website.json')
     giz_df = giz_df.rename(columns={'content':'project_description'})
+    # Sample 10 random rows for quick embeddings (seed set for reproducibility)
+    giz_df = giz_df.sample(n=5, random_state=42)    
     giz_df['text_size'] = giz_df.apply(lambda x: len((x['project_name'] + x['project_description']).split()), axis=1)
     giz_df['chunks'] = giz_df.apply(lambda x:create_chunks(x['project_name'] + x['project_description']),axis=1)
     print("initial df length:",len(giz_df))