Upload clean.py

clean.py

@@ -225,7 +225,7 @@ def calculate_nonconforming_cells(df):
     return nonconforming_cells
 
 
-def clean_data(df):
+def clean_data(df, primary_key_column):
     start_time = time.time()
     process_times = {}
     removed_rows = 0
@@ -267,18 +267,19 @@ def clean_data(df):
     # Step 5: Clean columns (in batches)
     column_cleaning_times = {}
     for i, column in enumerate(df.columns):
-        column_start_time = time.time()
-        df, nonconforming = clean_column(df, column)
-        column_cleaning_times[f"Clean column: {column}"] = time.time() - column_start_time
-        yield (5 + i) / total_steps, f"Cleaning column: {column}"
+        if column != primary_key_column:
+            column_start_time = time.time()
+            df, nonconforming = clean_column(df, column)
+            column_cleaning_times[f"Clean column: {column}"] = time.time() - column_start_time
+            yield (5 + i) / total_steps, f"Cleaning column: {column}"
     process_times.update(column_cleaning_times)
 
     # Step 6: Remove outliers from numeric columns
     step_start_time = time.time()
-    df, outlier_rows_removed = remove_outliers(df)
+    df, outlier_rows_removed = remove_outliers(df, primary_key_column)
     removed_rows += outlier_rows_removed
     process_times['Remove outliers'] = time.time() - step_start_time
     yield 1.0, (df, nonconforming_cells_before, process_times, removed_columns, removed_rows)
 
     print("Cleaning process completed.")
-    print_dataframe_info(df, "Final - ")
+    print_dataframe_info(df, "Final - ")