Change data source

app.py

@@ -24,39 +24,49 @@ def create_chunks(text):
     return texts 
 
 def get_chunks():
-    orgas_df = pd.read_csv("iati_files/project_orgas.csv")
-    region_df = pd.read_csv("iati_files/project_region.csv")
-    sector_df = pd.read_csv("iati_files/project_sector.csv")
-    status_df = pd.read_csv("iati_files/project_status.csv")
-    texts_df = pd.read_csv("iati_files/project_texts.csv")
-
-    projects_df = pd.merge(orgas_df, region_df, on='iati_id', how='inner')
-    projects_df = pd.merge(projects_df, sector_df, on='iati_id', how='inner')
-    projects_df = pd.merge(projects_df, status_df, on='iati_id', how='inner')
-    projects_df = pd.merge(projects_df, texts_df, on='iati_id', how='inner')
-    giz_df = projects_df[projects_df.client.str.contains('bmz')].reset_index(drop=True)
-
-    giz_df.drop(columns= ['orga_abbreviation', 'client',
-        'orga_full_name', 'country', 
-        'country_flag', 'crs_5_code', 'crs_3_code',
-        'sgd_pred_code'], inplace=True)
-
-    giz_df['text_size'] = giz_df.apply(lambda x: len((x['title_main'] + x['description_main']).split()), axis=1)
-    giz_df['chunks'] = giz_df.apply(lambda x:create_chunks(x['title_main'] + x['description_main']),axis=1)
+    #orgas_df = pd.read_csv("iati_files/project_orgas.csv")
+    #region_df = pd.read_csv("iati_files/project_region.csv")
+    #sector_df = pd.read_csv("iati_files/project_sector.csv")
+    #status_df = pd.read_csv("iati_files/project_status.csv")
+    #texts_df = pd.read_csv("iati_files/project_texts.csv")
+
+    #projects_df = pd.merge(orgas_df, region_df, on='iati_id', how='inner')
+    #projects_df = pd.merge(projects_df, sector_df, on='iati_id', how='inner')
+    #projects_df = pd.merge(projects_df, status_df, on='iati_id', how='inner')
+    #projects_df = pd.merge(projects_df, texts_df, on='iati_id', how='inner')
+    #giz_df = projects_df[projects_df.client.str.contains('bmz')].reset_index(drop=True)
+
+    #giz_df.drop(columns= ['orga_abbreviation', 'client',
+    #    'orga_full_name', 'country', 
+    #    'country_flag', 'crs_5_code', 'crs_3_code',
+    #    'sgd_pred_code'], inplace=True)
+
+    giz_df = pd.read_json('iati_files/data_giz_website.json')
+    giz_df = giz_df.rename(columns={'content':'project_description'})
+    
+
+    giz_df['text_size'] = giz_df.apply(lambda x: len((x['project_name'] + x['project_description']).split()), axis=1)
+    giz_df['chunks'] = giz_df.apply(lambda x:create_chunks(x['project_name'] + x['project_description']),axis=1)
     giz_df = giz_df.explode(column=['chunks'], ignore_index=True)
 
         
     placeholder= []
     for i in range(len(giz_df)):
         placeholder.append(Document(page_content= giz_df.loc[i,'chunks'], 
-                                metadata={"iati_id": giz_df.loc[i,'iati_id'],
-                                        "iati_orga_id":giz_df.loc[i,'iati_orga_id'],
-                                        "country_name":str(giz_df.loc[i,'country_name']),
-                                        "crs_5_name": giz_df.loc[i,'crs_5_name'],
-                                        "crs_3_name": giz_df.loc[i,'crs_3_name'],
-                                        "sgd_pred_str":giz_df.loc[i,'sgd_pred_str'],
-                                        "status":giz_df.loc[i,'status'],
-                                        "title_main":giz_df.loc[i,'title_main'],}))
+                                metadata={
+                                        "title_main":giz_df.loc[i,'title_main'],
+                                        "country_name":str(giz_df.loc[i,'countries']),
+                                        "client": giz_df_new.loc[i, 'client'],
+                                        "language":giz_df_new.loc[i, 'language'],
+                                        "political_sponsor":giz_df.loc[i, 'poli_trager'],
+                                        "url": giz_df.loc[i, 'url']
+                                        #"iati_id": giz_df.loc[i,'iati_id'],
+                                        #"iati_orga_id":giz_df.loc[i,'iati_orga_id'],
+                                        #"crs_5_name": giz_df.loc[i,'crs_5_name'],
+                                        #"crs_3_name": giz_df.loc[i,'crs_3_name'],
+                                        #"sgd_pred_str":giz_df.loc[i,'sgd_pred_str'],
+                                        #"status":giz_df.loc[i,'status'],
+                                        }))
     return placeholder
 
 def embed_chunks(chunks):