Update app.py

app.py

@@ -64,18 +64,33 @@ def load_and_concat_data():
     dataset_files = api.list_repo_files(repo_id=f"{HF_USERNAME}/{DATASET_NAME}", repo_type="dataset")
     feather_files = [file for file in dataset_files if file.endswith('.feather')]
 
-    all_data = []
-    for file in feather_files:
+    # Function to download and load a single file
+    def download_and_load(file):
         try:
-            file_content = api.hf_hub_download(repo_id=f"{HF_USERNAME}/{DATASET_NAME}", filename=file, repo_type="dataset", token=HF_TOKEN)
-            df = feather.read_feather(file_content)
-            all_data.append(df)
-        except Exception:
-            pass  # Silently skip files that can't be processed
+            file_content = api.hf_hub_download(
+                repo_id=f"{HF_USERNAME}/{DATASET_NAME}", 
+                filename=file, 
+                repo_type="dataset", 
+                token=HF_TOKEN
+            )
+            return feather.read_feather(file_content)
+        except Exception as e:
+            print(f"Error loading {file}: {str(e)}")
+            return None
+
+    # Download files in parallel
+    all_data = []
+    with ThreadPoolExecutor(max_workers=10) as executor:
+        future_to_file = {executor.submit(download_and_load, file): file for file in feather_files}
+        for future in as_completed(future_to_file):
+            df = future.result()
+            if df is not None:
+                all_data.append(df)
 
     if not all_data:
         return pd.DataFrame()
 
+    # Rest of your processing logic remains the same
     concatenated_df = pd.concat(all_data, ignore_index=True)
     
     columns_to_keep = [
@@ -85,31 +100,22 @@ def load_and_concat_data():
     filtered_df = concatenated_df[columns_to_keep].reset_index(drop=True)
     filtered_df['date_posted'] = pd.to_datetime(filtered_df['date_posted'], errors='coerce')
     
-    # Drop duplicates and rows with NaT in date_posted removed this to make it clear (jan13th)
-    #filtered_df = filtered_df.drop_duplicates().dropna(subset=['date_posted'])
-    #filtering based on data in 2024
     filtered_df = filtered_df[filtered_df['date_posted'].dt.year==2025]
-    # Convert titles and company name to lowercase
     filtered_df['title'] = filtered_df['title'].str.lower()
     filtered_df['company'] = filtered_df['company'].str.lower()
     
-    # Function to clean the location
     def clean_location(location):
         if pd.isna(location):
-            return location  # Return NaN as is
-        # Convert to lowercase
+            return location
         location = location.lower()
-        # Remove ', us' or ', usa' from the end using regex
         location = re.sub(r',\s*(us|usa)$', '', location)
         return location
 
-    # Clean the location in place
     filtered_df['location'] = filtered_df['location'].apply(clean_location)
-    #added new line to drop duplicate records
     filtered_df = filtered_df.drop_duplicates()
     
     return filtered_df
-    
+  
 @st.cache_data()
 def get_unique_values(df):
     return {