Upload clean.py

clean.py

@@ -155,11 +155,11 @@ def clean_column(df, column_name):
 
  # Convert column to determined data type
  if data_type == "float":
- df.loc[:, column_name] = pd.to_numeric(df[column_name], errors='coerce')
+ df[column_name] = pd.to_numeric(df[column_name], errors='coerce')
  elif data_type == "integer":
- df.loc[:, column_name] = pd.to_numeric(df[column_name], errors='coerce').astype('Int64')
+ df[column_name] = pd.to_numeric(df[column_name], errors='coerce').astype('Int64')
  elif data_type == "date":
- df[column_name] = pd.to_datetime(df[column_name], errors='coerce')
+ df[column_name] = pd.to_datetime(df[column_name], errors='coerce', dayfirst=True)
  elif data_type == "string" or data_type == "object":
  # Transform string values
  transform_result = transform_string_column(column_data, column_name)
@@ -178,8 +178,10 @@ def clean_column(df, column_name):
  print(f" Potential typos found: {typo_result['typos']}")
 
  # Set empty and invalid cells to NaN
- df.loc[empty_indices + invalid_indices, column_name] = np.nan
- nonconforming_cells = len(empty_indices) + len(invalid_indices)
+ indices_to_set_nan = set(empty_indices + invalid_indices)
+ existing_indices = df.index.intersection(indices_to_set_nan)
+ df.loc[existing_indices, column_name] = np.nan
+ nonconforming_cells = len(existing_indices)
 
  return df, nonconforming_cells