Upload report.py

report.py

@@ -23,7 +23,9 @@ def plot_heatmap(df, title):
 
 
 def plot_valid_data_percentage(original_df, cleaned_df, primary_key_column):
- columns_to_plot = [col for col in original_df.columns if col != primary_key_column]
+ # Get columns present in both DataFrames, excluding the primary key
+ columns_to_plot = [col for col in original_df.columns if col in cleaned_df.columns and col != primary_key_column]
+
  original_valid = (original_df[columns_to_plot].notna().sum() / len(original_df)) * 100
  cleaned_valid = (cleaned_df[columns_to_plot].notna().sum() / len(cleaned_df)) * 100
 
@@ -52,6 +54,13 @@ def plot_valid_data_percentage(original_df, cleaned_df, primary_key_column):
  plt.savefig(os.path.join(REPORT_DIR, 'valid_data_percentage.png'))
  plt.close()
 
+ # Print information about removed columns
+ removed_columns = set(original_df.columns) - set(cleaned_df.columns)
+ if removed_columns:
+ print(f"The following columns were removed during the cleaning process: {', '.join(removed_columns)}")
+ else:
+ print("No columns were removed during the cleaning process.")
+
 
 def plot_column_schemas(df):
  schemas = df.dtypes.astype(str).value_counts()