Update app.py

app.py

@@ -66,46 +66,64 @@ def load_and_concat_data():
     for file in csv_files:
         try:
             file_content = api.hf_hub_download(repo_id=f"{HF_USERNAME}/{DATASET_NAME}", filename=file, repo_type="dataset", token=HF_TOKEN)
-            df = pd.read_csv(file_content, engine='pyarrow')
-            all_data.append(df)
-        except Exception:
-            pass  # Silently skip files that can't be processed
+            
+            # Use PyArrow to read CSV
+            read_options = csv.ReadOptions(column_names=[
+                'site', 'job_url', 'title', 'company', 'location',
+                'job_type', 'date_posted', 'is_remote', 'company_url'
+            ])
+            parse_options = csv.ParseOptions(delimiter=',')
+            convert_options = csv.ConvertOptions(
+                timestamp_parsers=['%Y-%m-%d']
+            )
+            
+            table = csv.read_csv(file_content, read_options=read_options, parse_options=parse_options, convert_options=convert_options)
+            all_data.append(table)
+        except Exception as e:
+            print(f"Error processing file {file}: {str(e)}")
 
     if not all_data:
-        return pd.DataFrame()
+        return pa.Table.from_pandas(pd.DataFrame())
 
-    concatenated_df = pd.concat(all_data, ignore_index=True)
-    
-    columns_to_keep = [
-        'site', 'job_url', 'title', 'company', 'location',
-        'job_type', 'date_posted', 'is_remote', 'company_url'
-    ]
-    filtered_df = concatenated_df[columns_to_keep].reset_index(drop=True)
-    filtered_df['date_posted'] = pd.to_datetime(filtered_df['date_posted'], errors='coerce')
-    
-    # Drop duplicates and rows with NaT in date_posted
-    filtered_df = filtered_df.drop_duplicates().dropna(subset=['date_posted'])
-    #filtering based on data in 2024
-    filtered_df = filtered_df[filtered_df['date_posted'].dt.year==2024]
-    # Convert titles and company name to lowercase
-    filtered_df['title'] = filtered_df['title'].str.lower()
-    filtered_df['company'] = filtered_df['company'].str.lower()
-    
-    # Function to clean the location
+    # Concatenate all tables
+    concatenated_table = pa.concat_tables(all_data)
+
+    # Filter for 2024 data
+    mask = pc.year(concatenated_table['date_posted']) == 2024
+    filtered_table = concatenated_table.filter(mask)
+
+    # Convert titles and company names to lowercase
+    filtered_table = filtered_table.set_column(
+        filtered_table.schema.get_field_index('title'),
+        'title',
+        pc.utf8_lower(filtered_table['title'])
+    )
+    filtered_table = filtered_table.set_column(
+        filtered_table.schema.get_field_index('company'),
+        'company',
+        pc.utf8_lower(filtered_table['company'])
+    )
+
+    # Clean location
     def clean_location(location):
-        if pd.isna(location):
-            return location  # Return NaN as is
-        # Convert to lowercase
+        if location is None:
+            return None
         location = location.lower()
-        # Remove ', us' or ', usa' from the end using regex
-        location = re.sub(r',\s*(us|usa)$', '', location)
-        return location
-
-    # Clean the location in place
-    filtered_df['location'] = filtered_df['location'].apply(clean_location)
-    #added new line to drop duplciate records
-    filtered_df = filtered_df.drop_duplicates()
-    
+        return re.sub(r',\s*(us|usa)$', '', location)
+
+    cleaned_locations = pc.map(filtered_table['location'], clean_location)
+    filtered_table = filtered_table.set_column(
+        filtered_table.schema.get_field_index('location'),
+        'location',
+        cleaned_locations
+    )
+
+    # Remove duplicates
+    filtered_table = filtered_table.group_by(filtered_table.column_names).aggregate([])
+
+    # Convert to pandas DataFrame for compatibility with the rest of your code
+    filtered_df = filtered_table.to_pandas()
+
     return filtered_df
     
 @st.cache_data()