Upload 5 files

app.py

@@ -0,0 +1,93 @@
+import gradio as gr
+import pandas as pd
+from clean import clean_data
+from report import create_full_report, REPORT_DIR
+import os
+import tempfile
+
+def clean_and_visualize(file, progress=gr.Progress()):
+ # Load the data
+ df = pd.read_csv(file.name)
+
+ # Clean the data
+ cleaned_df = None
+ nonconforming_cells_before = None
+ process_times = None
+ removed_columns = None
+ removed_rows = None
+
+ for progress_value, status_text in clean_data(df):
+ if isinstance(status_text, tuple):
+ cleaned_df, nonconforming_cells_before, process_times, removed_columns, removed_rows = status_text
+ progress(progress_value, desc="Cleaning completed")
+ else:
+ progress(progress_value, desc=status_text)
+
+ # Generate full visualization report
+ create_full_report(
+ df,
+ cleaned_df,
+ nonconforming_cells_before,
+ process_times,
+ removed_columns,
+ removed_rows
+ )
+
+ # Save cleaned DataFrame to a temporary CSV file
+ with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as tmp_file:
+ cleaned_df.to_csv(tmp_file.name, index=False)
+ cleaned_csv_path = tmp_file.name
+
+ # Collect all generated images
+ image_files = [os.path.join(REPORT_DIR, f) for f in os.listdir(REPORT_DIR) if f.endswith('.png')]
+
+ return cleaned_csv_path, image_files
+
+def launch_app():
+ with gr.Blocks() as app:
+ gr.Markdown("# Data Cleaning and Visualization App")
+
+ with gr.Row():
+ file_input = gr.File(label="Upload CSV File")
+
+ with gr.Row():
+ clean_button = gr.Button("Start Cleaning")
+
+ with gr.Row():
+ progress_bar = gr.Progress()
+
+ with gr.Row():
+ download_button = gr.Button("Download Cleaned CSV", visible=False)
+ cleaned_file_output = gr.File(label="Cleaned CSV", visible=False)
+
+ with gr.Row():
+ output_gallery = gr.Gallery(label="Visualization Results", show_label=True, elem_id="gallery", columns=[2],
+ rows=[2], object_fit="contain", height="auto")
+
+ def process_and_show_download(file):
+ cleaned_csv_path, image_files = clean_and_visualize(file, progress=progress_bar)
+ return (
+ gr.Button.update(visible=True),
+ gr.File.update(value=cleaned_csv_path, visible=True),
+ image_files
+ )
+
+ clean_button.click(
+ fn=process_and_show_download,
+ inputs=file_input,
+ outputs=[download_button, cleaned_file_output, output_gallery]
+ )
+
+ def trigger_download():
+ return gr.File.update(visible=True)
+
+ download_button.click(
+ fn=trigger_download,
+ inputs=[],
+ outputs=[cleaned_file_output]
+ )
+
+ app.launch()
+
+if __name__ == "__main__":
+ launch_app()

clean.py

@@ -0,0 +1,295 @@
+import pandas as pd
+import numpy as np
+import json
+import time
+from tqdm import tqdm
+from llm_config import generate_llm_response
+from llm_prompts import (
+ CHECK_HEADERS_PROMPT,
+ NORMALIZE_HEADERS_PROMPT,
+ CHECK_COLUMN_CONTENT_PROMPT,
+ CHECK_TYPOS_PROMPT,
+ TRANSFORM_STRING_PROMPT,
+ CHECK_LOW_COUNT_VALUES_PROMPT
+)
+
+BATCH_SIZE = 50
+EMPTY_THRESHOLD = 0.5
+
+
+def print_dataframe_info(df, step=""):
+ num_columns = df.shape[1]
+ num_rows = df.shape[0]
+ num_cells = num_columns * num_rows
+ print(f"{step}Dataframe info:")
+ print(f" Number of columns: {num_columns}")
+ print(f" Number of rows: {num_rows}")
+ print(f" Total number of cells: {num_cells}")
+
+
+def check_and_normalize_column_headers(df):
+ print("Checking and normalizing column headers...")
+
+ check_prompt = CHECK_HEADERS_PROMPT.format(columns=df.columns.tolist())
+ check_response = generate_llm_response(check_prompt)
+ try:
+ invalid_columns = json.loads(check_response)
+ if invalid_columns:
+ print(f"Columns with invalid names (indices): {invalid_columns}")
+ for idx in invalid_columns:
+ new_name = f"column_{idx}"
+ print(f"Renaming column at index {idx} to '{new_name}'")
+ df.rename(columns={df.columns[idx]: new_name}, inplace=True)
+ else:
+ print("All column headers are valid or no invalid headers detected.")
+ except json.JSONDecodeError:
+ print("Error parsing LLM response for column headers check.")
+
+ normalize_prompt = NORMALIZE_HEADERS_PROMPT.format(columns=df.columns.tolist())
+ normalize_response = generate_llm_response(normalize_prompt)
+ try:
+ normalized_names = json.loads(normalize_response)
+ if normalized_names:
+ df.rename(columns=normalized_names, inplace=True)
+ print("Column names have been normalized.")
+ else:
+ print("No column names were normalized. Proceeding with current names.")
+ except json.JSONDecodeError:
+ print("Error parsing LLM response for column name normalization.")
+
+ # Fallback normalization
+ df.columns = [col.lower().replace(' ', '_') for col in df.columns]
+ print("Applied fallback normalization to ensure valid column names.")
+
+ return df
+
+
+def process_column_batch(column_data, column_name):
+ sample = column_data.sample(n=min(BATCH_SIZE, len(column_data)), random_state=42).tolist()
+ prompt = CHECK_COLUMN_CONTENT_PROMPT.format(column_name=column_name, sample_values=str(sample))
+ response = generate_llm_response(prompt)
+ try:
+ result = json.loads(response)
+ if not all(key in result for key in ['data_type', 'empty_indices', 'invalid_indices']):
+ raise ValueError("Missing required keys in LLM response")
+ return result
+ except (json.JSONDecodeError, ValueError) as e:
+ print(f"Error parsing LLM response for column {column_name}: {str(e)}")
+ print(f"LLM Response: {response}")
+ return {'data_type': 'string', 'empty_indices': [], 'invalid_indices': []}
+
+
+def check_typos(column_data, column_name):
+ sample = column_data.sample(n=min(BATCH_SIZE, len(column_data)), random_state=42).tolist()
+ prompt = CHECK_TYPOS_PROMPT.format(column_name=column_name, sample_values=str(sample))
+ response = generate_llm_response(prompt)
+ try:
+ return json.loads(response)
+ except json.JSONDecodeError:
+ print(f"Error parsing LLM response for typo check in column {column_name}")
+ return {"typos": {}}
+
+
+def transform_string_column(column_data, column_name):
+ unique_values = column_data.unique().tolist()
+ prompt = TRANSFORM_STRING_PROMPT.format(column_name=column_name, unique_values=unique_values)
+ response = generate_llm_response(prompt)
+ try:
+ result = json.loads(response)
+ return result
+ except json.JSONDecodeError:
+ print(f"Error parsing LLM response for string transformation in column {column_name}")
+ return {}
+
+
+def check_low_count_values(column_data, column_name):
+ value_counts = column_data.value_counts().to_dict()
+ prompt = CHECK_LOW_COUNT_VALUES_PROMPT.format(column_name=column_name, value_counts=value_counts)
+ response = generate_llm_response(prompt)
+ try:
+ result = json.loads(response)
+ return result
+ except json.JSONDecodeError:
+ print(f"Error parsing LLM response for low count values in column {column_name}")
+ return []
+
+
+def remove_empty_columns(df, threshold=EMPTY_THRESHOLD):
+ print(f"Removing columns with less than {threshold * 100}% valid data...")
+ valid_threshold = int(df.shape[0] * threshold)
+ df = df.dropna(axis=1, thresh=valid_threshold)
+ return df
+
+
+def remove_empty_rows(df, threshold=EMPTY_THRESHOLD):
+ print(f"Removing rows with less than {threshold * 100}% valid data...")
+ valid_threshold = int(df.shape[1] * threshold)
+ df = df.dropna(axis=0, thresh=valid_threshold)
+ return df
+
+
+def remove_low_count_categories(df):
+ print("Removing strings with count below 2...")
+ for col in df.select_dtypes(include=['object']).columns:
+ value_counts = df[col].value_counts()
+ to_remove = value_counts[value_counts < 2].index
+ df[col] = df[col].replace(to_remove, np.nan)
+ return df
+
+
+def clean_column(df, column_name):
+ print(f"Cleaning column: {column_name}")
+ column_data = df[column_name]
+ total_rows = len(column_data)
+ empty_indices = []
+ invalid_indices = []
+ data_type = "string"
+ nonconforming_cells = 0
+
+ for i in range(0, total_rows, BATCH_SIZE):
+ batch = column_data.iloc[i:i + BATCH_SIZE]
+ result = process_column_batch(batch, column_name)
+
+ valid_empty_indices = [idx for idx in result["empty_indices"] if idx + i < total_rows]
+ valid_invalid_indices = [idx for idx in result["invalid_indices"] if idx + i < total_rows]
+
+ empty_indices.extend([idx + i for idx in valid_empty_indices])
+ invalid_indices.extend([idx + i for idx in valid_invalid_indices])
+
+ if i == 0: # Use the data type from the first batch
+ data_type = result["data_type"]
+
+ print(f" Data type determined: {data_type}")
+ print(f" Empty cells: {len(empty_indices)}")
+ print(f" Invalid cells: {len(invalid_indices)}")
+
+ # Convert column to determined data type
+ if data_type == "float":
+ df.loc[:, column_name] = pd.to_numeric(df[column_name], errors='coerce')
+ elif data_type == "integer":
+ df.loc[:, column_name] = pd.to_numeric(df[column_name], errors='coerce').astype('Int64')
+ elif data_type == "date":
+ df[column_name] = pd.to_datetime(df[column_name], errors='coerce')
+ elif data_type == "string" or data_type == "object":
+ # Transform string values
+ transform_result = transform_string_column(column_data, column_name)
+ df[column_name] = df[column_name].map(transform_result).fillna(df[column_name])
+
+ # Handle "nan" strings
+ df[column_name] = df[column_name].replace({"nan": np.nan, "NaN": np.nan, "NAN": np.nan})
+
+ # Check for low count values
+ low_count_values = check_low_count_values(df[column_name], column_name)
+ df.loc[df[column_name].isin(low_count_values), column_name] = np.nan
+
+ # Check for typos
+ typo_result = check_typos(df[column_name], column_name)
+ if typo_result["typos"]:
+ print(f" Potential typos found: {typo_result['typos']}")
+
+ # Set empty and invalid cells to NaN
+ df.loc[empty_indices + invalid_indices, column_name] = np.nan
+ nonconforming_cells = len(empty_indices) + len(invalid_indices)
+
+ return df, nonconforming_cells
+
+
+def remove_outliers(df):
+ print("Removing rows with outliers from numeric/integer/float columns...")
+ rows_to_remove = set()
+ for column in df.select_dtypes(include=[np.number]).columns:
+ q1 = df[column].quantile(0.25)
+ q3 = df[column].quantile(0.75)
+ iqr = q3 - q1
+ lower_bound = q1 - 1.5 * iqr
+ upper_bound = q3 + 1.5 * iqr
+ outlier_rows = df[(df[column] < lower_bound) | (df[column] > upper_bound)].index
+ rows_to_remove.update(outlier_rows)
+
+ initial_rows = len(df)
+ df = df.drop(index=list(rows_to_remove))
+ removed_rows = initial_rows - len(df)
+ print(f"Removed {removed_rows} rows containing outliers.")
+ return df, removed_rows
+
+
+def calculate_nonconforming_cells(df):
+ nonconforming_cells = {}
+ for column in df.columns:
+ # Count NaN values
+ nan_count = df[column].isna().sum()
+
+ # For numeric columns, count infinite values
+ if np.issubdtype(df[column].dtype, np.number):
+ inf_count = np.isinf(df[column]).sum()
+ else:
+ inf_count = 0
+
+ # For object columns, count empty strings
+ if df[column].dtype == 'object':
+ empty_string_count = (df[column] == '').sum()
+ else:
+ empty_string_count = 0
+
+ nonconforming_cells[column] = nan_count + inf_count + empty_string_count
+
+ return nonconforming_cells
+
+
+def clean_data(df):
+ start_time = time.time()
+ process_times = {}
+ removed_rows = 0
+ removed_columns = 0
+
+ print("Starting data validation and cleaning...")
+ print_dataframe_info(df, "Initial - ")
+
+ # Calculate nonconforming cells before cleaning
+ nonconforming_cells_before = calculate_nonconforming_cells(df)
+
+ steps = ['Normalize headers', 'Remove empty columns', 'Remove empty rows', 'Remove low count strings', 'Clean columns', 'Remove outliers']
+ total_steps = len(steps) + len(df.columns) # Add column count for individual column cleaning
+
+ # Step 1: Normalize column headers
+ step_start_time = time.time()
+ df = check_and_normalize_column_headers(df)
+ process_times['Normalize headers'] = time.time() - step_start_time
+ yield 1 / total_steps, "Normalized headers"
+
+ # Step 2: Remove empty columns (less than 60% valid data)
+ step_start_time = time.time()
+ df = remove_empty_columns(df)
+ process_times['Remove empty columns'] = time.time() - step_start_time
+ yield 2 / total_steps, "Removed empty columns"
+
+ # Step 3: Remove empty rows (less than 60% valid data)
+ step_start_time = time.time()
+ df = remove_empty_rows(df)
+ process_times['Remove empty rows'] = time.time() - step_start_time
+ yield 3 / total_steps, "Removed empty rows"
+
+ # Step 4: Remove low count categories
+ step_start_time = time.time()
+ df = remove_low_count_categories(df)
+ process_times['Remove low count strings'] = time.time() - step_start_time
+ yield 4 / total_steps, "Removed low count strings"
+
+ # Step 5: Clean columns (in batches)
+ column_cleaning_times = {}
+ for i, column in enumerate(df.columns):
+ column_start_time = time.time()
+ df, nonconforming = clean_column(df, column)
+ column_cleaning_times[f"Clean column: {column}"] = time.time() - column_start_time
+ yield (5 + i) / total_steps, f"Cleaning column: {column}"
+ process_times.update(column_cleaning_times)
+
+ # Step 6: Remove outliers from numeric columns
+ step_start_time = time.time()
+ df, outlier_rows_removed = remove_outliers(df)
+ removed_rows += outlier_rows_removed
+ process_times['Remove outliers'] = time.time() - step_start_time
+ yield 1.0, (df, nonconforming_cells_before, process_times, removed_columns, removed_rows)
+
+ print("Cleaning process completed.")
+ print_dataframe_info(df, "Final - ")

llm_prompts.py

@@ -0,0 +1,123 @@
+CHECK_HEADERS_PROMPT = """
+Analyze the following DataFrame columns and identify any columns without names or with invalid names. 
+Return only a JSON list of column indices (0-based) that need attention, without any explanation.
+Columns: {columns}
+"""
+
+NORMALIZE_HEADERS_PROMPT = """
+Analyze the following DataFrame column names and normalize them according to these rules:
+1. Convert to lowercase
+2. Replace empty strings or spaces with underscores
+3. Remove any invalid characters (keep only letters, numbers, and underscores)
+
+Return only a JSON object where keys are the original column names and values are the normalized names, without any explanation.
+Column names: {columns}
+"""
+
+CHECK_COLUMN_CONTENT_PROMPT = """
+Analyze the following sample of values from the column '{column_name}' and determine:
+1. The most appropriate data type (float, integer, string, or date)
+2. Indices of empty or blank values
+3. Indices of values that don't conform to the determined data type
+
+Sample values:
+{sample_values}
+
+Return only a JSON object with the following structure, without any explanation:
+{{
+ "data_type": "detected_type",
+ "empty_indices": [list of indices of empty or blank values],
+ "invalid_indices": [list of indices of values that don't conform to the detected type]
+}}
+"""
+
+CHECK_TYPOS_PROMPT = """
+Analyze the following sample of values from the column '{column_name}' and identify any potential typos or misspellings.
+For each identified typo, suggest a correction.
+
+Sample values:
+{sample_values}
+
+Return only a JSON object with the following structure, without any explanation:
+{{
+ "typos": {{
+ "original_value1": "corrected_value1",
+ "original_value2": "corrected_value2",
+ ...
+ }}
+}}
+
+If no typos are found, return an empty object for "typos".
+"""
+
+ENCODE_STRING_PROMPT = """
+Analyze the following unique values from the column '{column_name}' and create an encoding scheme.
+Assign a unique integer to each unique string value, starting from 0.
+
+Unique values:
+{unique_values}
+
+Return only a JSON object with the following structure, without any explanation:
+{{
+ "string_value1": 0,
+ "string_value2": 1,
+ "string_value3": 2,
+ ...
+}}
+
+Ensure that each unique string value is assigned a unique integer.
+"""
+
+DETERMINE_DTYPE_PROMPT = """
+Analyze the following sample values from a column and determine the most appropriate data type.
+Possible types are: float, integer, string, or date.
+If more than 80% of the values conform to a specific type, choose that type.
+Otherwise, default to string.
+
+Sample values:
+{sample_values}
+
+Return only a JSON object with the following structure, without any explanation:
+{{
+ "column_type": "detected_type",
+ "invalid_indices": [list of indices that do not conform to the detected type]
+}}
+"""
+
+TRANSFORM_STRING_PROMPT = """
+Transform the following unique string values from the column '{column_name}' to lowercase.
+If a value is a variation of "nan" (case-insensitive), map it to "nan".
+
+Unique values:
+{unique_values}
+
+Return only a JSON object with the following structure, without any explanation:
+{{
+ "original_value1": "transformed_value1",
+ "original_value2": "transformed_value2",
+ ...
+}}
+"""
+
+CHECK_LOW_COUNT_VALUES_PROMPT = """
+Analyze the following value counts from the column '{column_name}' and identify values with a count lower than 2.
+
+Value counts:
+{value_counts}
+
+Return only a JSON list of values that have a count lower than 2, without any explanation.
+"""
+
+
+CHECK_SCHEMA_CONFORMITY_PROMPT = """
+Analyze the following sample of values from the column '{column_name}' and check if they conform to the determined data type '{data_type}'.
+
+Sample values:
+{sample_values}
+
+Return only a JSON object with the following structure, without any explanation:
+{{
+ "conforming_indices": [list of indices of values that conform to the data type],
+ "nonconforming_indices": [list of indices of values that do not conform to the data type]
+}}
+"""

manage_schema.py

@@ -0,0 +1,51 @@
+import pandas as pd
+import numpy as np
+import json
+from llm_config import generate_llm_response
+from llm_prompts import DETERMINE_DTYPE_PROMPT
+
+SAMPLE_SIZE = 200
+
+
+def determine_column_type(df, column):
+ sample = df[column].sample(n=min(SAMPLE_SIZE, len(df)), random_state=42).tolist()
+ prompt = DETERMINE_DTYPE_PROMPT.format(sample_values=str(sample))
+ response = generate_llm_response(prompt)
+
+ try:
+ result = json.loads(response)
+ return result['column_type'], result['invalid_indices']
+ except (json.JSONDecodeError, KeyError):
+ print(f"Error parsing LLM response for column {column}")
+ return 'string', []
+
+
+def enforce_column_type(df, column, column_type, invalid_indices):
+ if column_type == 'float':
+ df[column] = pd.to_numeric(df[column], errors='coerce')
+ elif column_type == 'integer':
+ df[column] = pd.to_numeric(df[column], errors='coerce').astype('Int64')
+ elif column_type == 'date':
+ df[column] = pd.to_datetime(df[column], errors='coerce')
+
+ # Set invalid values to NaN
+ df.loc[invalid_indices, column] = np.nan
+
+ return df
+
+
+def process_dataframe(df):
+ print("Determining and enforcing column data types...")
+
+ for column in df.columns:
+ print(f"\nProcessing column: {column}")
+ column_type, invalid_indices = determine_column_type(df, column)
+ print(f" Detected type: {column_type}")
+ print(f" Number of invalid values: {len(invalid_indices)}")
+
+ df = enforce_column_type(df, column, column_type, invalid_indices)
+
+ valid_percentage = (df[column].count() / len(df)) * 100
+ print(f" Percentage of valid values after type enforcement: {valid_percentage:.2f}%")
+
+ return df

report.py

@@ -0,0 +1,208 @@
+import os
+import numpy as np
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+from datetime import datetime
+
+REPORT_DIR = f"cleaning_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+os.makedirs(REPORT_DIR, exist_ok=True)
+
+def save_plot(fig, filename):
+ fig.savefig(os.path.join(REPORT_DIR, filename), dpi=400, bbox_inches='tight')
+ plt.close(fig)
+
+def plot_heatmap(df, title):
+ plt.figure(figsize=(12, 8))
+ sns.heatmap(df.isnull(), cbar=False, cmap='viridis')
+ plt.title(title)
+ plt.tight_layout()
+ save_plot(plt.gcf(), f'{title.lower().replace(" ", "_")}.png')
+
+
+def plot_valid_data_percentage(original_df, cleaned_df):
+ original_valid = (original_df.notna().sum() / len(original_df)) * 100
+ cleaned_valid = (cleaned_df.notna().sum() / len(cleaned_df)) * 100
+
+ # Create a DataFrame with both original and cleaned percentages
+ combined_df = pd.DataFrame({
+ 'Original': original_valid,
+ 'Cleaned': cleaned_valid
+ }).fillna(0) # Fill NaN with 0 for columns that were removed
+
+ plt.figure(figsize=(15, 8))
+ combined_df.plot(kind='bar', width=0.8, alpha=0.8)
+
+ plt.xlabel('Columns')
+ plt.ylabel('Percentage of Valid Data')
+ plt.title('Percentage of Valid Data Before and After Cleaning')
+ plt.xticks(rotation=90)
+ plt.legend(['Before Cleaning', 'After Cleaning'])
+
+ # Add percentage labels on the bars
+ for i, (index, row) in enumerate(combined_df.iterrows()):
+ plt.text(i, row['Original'], f'{row["Original"]:.1f}%', ha='center', va='bottom')
+ if row['Cleaned'] > 0: # Only add label if column exists in cleaned data
+ plt.text(i, row['Cleaned'], f'{row["Cleaned"]:.1f}%', ha='center', va='bottom')
+
+ plt.tight_layout()
+ plt.savefig(os.path.join(REPORT_DIR, 'valid_data_percentage.png'))
+ plt.close()
+
+def plot_column_schemas(df):
+ schemas = df.dtypes.astype(str).value_counts()
+ fig, ax = plt.subplots(figsize=(10, 6))
+ sns.barplot(x=schemas.index, y=schemas.values, ax=ax)
+ ax.set_title('Column Data Types')
+ ax.set_xlabel('Data Type')
+ ax.set_ylabel('Count')
+ save_plot(fig, 'column_schemas.png')
+
+def plot_nonconforming_cells(nonconforming_cells):
+ # Ensure that nonconforming_cells is a dictionary
+ if isinstance(nonconforming_cells, dict):
+ # Proceed with plotting if it's a dictionary
+ fig, ax = plt.subplots(figsize=(12, 6))
+ sns.barplot(x=list(nonconforming_cells.keys()), y=list(nonconforming_cells.values()), ax=ax)
+ ax.set_title('Nonconforming Cells by Column')
+ ax.set_xlabel('Columns')
+ ax.set_ylabel('Number of Nonconforming Cells')
+ plt.xticks(rotation=90)
+ save_plot(fig, 'nonconforming_cells.png')
+ else:
+ print(f"Expected nonconforming_cells to be a dictionary, but got {type(nonconforming_cells)}.")
+
+
+def plot_column_distributions(original_df, cleaned_df):
+ numeric_columns = original_df.select_dtypes(include=[np.number]).columns
+ num_columns = len(numeric_columns)
+
+ if num_columns == 0:
+ print("No numeric columns found for distribution plots.")
+ return
+
+ # Create subplots for distributions
+ fig, axes = plt.subplots(nrows=(num_columns + 2) // 3, ncols=3, figsize=(18, 5 * ((num_columns + 2) // 3)))
+ axes = axes.flatten() if num_columns > 1 else [axes]
+
+ for i, column in enumerate(numeric_columns):
+ if column in cleaned_df.columns:
+ sns.histplot(original_df[column].dropna(), ax=axes[i], kde=True, color='blue', label='Before Cleaning', alpha=0.5)
+ sns.histplot(cleaned_df[column].dropna(), ax=axes[i], kde=True, color='orange', label='After Cleaning', alpha=0.5)
+ axes[i].set_title(f'{column} - Distribution Before & After Cleaning')
+ axes[i].legend()
+
+ # Remove any unused subplots
+ for j in range(i + 1, len(axes)):
+ fig.delaxes(axes[j])
+
+ plt.tight_layout()
+ save_plot(fig, 'distributions_before_after_cleaning.png')
+
+
+def plot_boxplot_with_outliers(df):
+ print("Plotting boxplots with outliers...")
+ numeric_columns = df.select_dtypes(include=[np.number]).columns
+ num_columns = len(numeric_columns)
+
+ if num_columns == 0:
+ print("No numeric columns found for boxplot.")
+ return
+
+ # Create subplots based on the number of numeric columns
+ fig, axes = plt.subplots(nrows=(num_columns + 2) // 3, ncols=3, figsize=(15, 5 * ((num_columns + 2) // 3)))
+ axes = axes.flatten() if num_columns > 1 else [axes]
+
+ for i, column in enumerate(numeric_columns):
+ sns.boxplot(x=df[column], ax=axes[i])
+ axes[i].set_title(f'Boxplot of {column} with Outliers')
+
+ # Remove any unused subplots
+ for j in range(i + 1, len(axes)):
+ fig.delaxes(axes[j])
+
+ plt.tight_layout()
+ save_plot(fig, 'boxplots_with_outliers.png')
+
+
+def plot_correlation_heatmap(df):
+ # Select only numeric, float, and integer columns
+ numeric_df = df.select_dtypes(include=[np.number])
+
+ # Compute the correlation matrix
+ correlation_matrix = numeric_df.corr()
+
+ # Plot the heatmap
+ fig, ax = plt.subplots(figsize=(15, 10))
+ sns.heatmap(correlation_matrix, annot=True, fmt=".2f", cmap='coolwarm', ax=ax, cbar_kws={'label': 'Correlation'})
+ ax.set_title('Correlation Heatmap')
+ save_plot(fig, 'correlation_heatmap.png')
+
+
+
+def plot_process_times(process_times):
+ # Convert seconds to minutes
+ process_times_minutes = {k: v / 60 for k, v in process_times.items()}
+
+ # Separate main processes and column cleaning processes
+ main_processes = {k: v for k, v in process_times_minutes.items() if not k.startswith("Clean column:")}
+ column_processes = {k: v for k, v in process_times_minutes.items() if k.startswith("Clean column:")}
+
+ # Create the plot
+ fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(15, 10))
+
+ # Plot main processes
+ bars1 = ax1.bar(main_processes.keys(), main_processes.values())
+ ax1.set_title('Main Process Times')
+ ax1.set_ylabel('Time (minutes)')
+ ax1.tick_params(axis='x', rotation=45)
+
+ # Plot column cleaning processes
+ bars2 = ax2.bar(column_processes.keys(), column_processes.values())
+ ax2.set_title('Column Cleaning Times')
+ ax2.set_ylabel('Time (minutes)')
+ ax2.tick_params(axis='x', rotation=90)
+
+ # Add value labels on top of each bar
+ for ax, bars in zip([ax1, ax2], [bars1, bars2]):
+ for bar in bars:
+ height = bar.get_height()
+ ax.text(bar.get_x() + bar.get_width() / 2., height,
+ f'{height:.2f}', ha='center', va='bottom')
+
+ # Add total time to the plot
+ total_time = sum(process_times_minutes.values())
+ fig.suptitle(f'Process Times (Total: {total_time:.2f} minutes)', fontsize=16)
+
+ plt.tight_layout()
+ save_plot(fig, 'process_times.png')
+
+
+def create_full_report(original_df, cleaned_df, nonconforming_cells_before, process_times, removed_columns, removed_rows):
+ os.makedirs(REPORT_DIR, exist_ok=True)
+
+ sns.set_style("whitegrid")
+ plt.rcParams['figure.dpi'] = 400
+
+ print("Plotting valid data percentages...")
+ plot_valid_data_percentage(original_df, cleaned_df)
+
+ print("Plotting column schemas...")
+ plot_column_schemas(cleaned_df)
+
+ print("Plotting nonconforming cells before cleaning...")
+ plot_nonconforming_cells(nonconforming_cells_before)
+
+ print("Plotting column distributions...")
+ plot_column_distributions(original_df, cleaned_df)
+
+ print("Plotting process times...")
+ plot_process_times(process_times)
+
+ print("Plotting heatmaps...")
+ plot_heatmap(original_df, "Missing Values Before Cleaning")
+
+ print("Plotting correlation heatmap...")
+ plot_correlation_heatmap(cleaned_df)
+
+ print(f"All visualization reports saved in directory: {REPORT_DIR}")