Spaces:
Running
Running
| import os | |
| import re | |
| import pandas as pd | |
| import gradio as gr | |
| import gzip | |
| import pickle | |
| def detect_file_type(filename): | |
| """Detect the file type based on its extension.""" | |
| if (filename.endswith('.csv')) | (filename.endswith('.csv.gz')) | (filename.endswith('.zip')): | |
| return 'csv' | |
| elif filename.endswith('.xlsx'): | |
| return 'xlsx' | |
| elif filename.endswith('.parquet'): | |
| return 'parquet' | |
| elif filename.endswith('.pkl.gz'): | |
| return 'pkl.gz' | |
| else: | |
| raise ValueError("Unsupported file type.") | |
| def read_file(filename): | |
| """Read the file based on its detected type.""" | |
| file_type = detect_file_type(filename) | |
| print("Loading in file") | |
| if file_type == 'csv': | |
| file = pd.read_csv(filename, low_memory=False).reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore") | |
| elif file_type == 'xlsx': | |
| file = pd.read_excel(filename).reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore") | |
| elif file_type == 'parquet': | |
| file = pd.read_parquet(filename).reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore") | |
| elif file_type == 'pkl.gz': | |
| with gzip.open(filename, 'rb') as file: | |
| file = pickle.load(file) | |
| #file = pd.read_pickle(filename) | |
| print("File load complete") | |
| return file | |
| def put_columns_in_df(in_file, in_bm25_column): | |
| ''' | |
| When file is loaded, update the column dropdown choices and change 'clean data' dropdown option to 'no'. | |
| ''' | |
| file_list = [string.name for string in in_file] | |
| data_file_names = [string.lower() for string in file_list if "npz" not in string.lower()] | |
| data_file_name = data_file_names[0] | |
| new_choices = [] | |
| concat_choices = [] | |
| df = read_file(data_file_name) | |
| new_choices = list(df.columns) | |
| concat_choices.extend(new_choices) | |
| return gr.Dropdown(choices=concat_choices), gr.Dropdown(choices=concat_choices), df | |
| def get_file_path_end(file_path): | |
| # First, get the basename of the file (e.g., "example.txt" from "/path/to/example.txt") | |
| basename = os.path.basename(file_path) | |
| # Then, split the basename and its extension and return only the basename without the extension | |
| filename_without_extension, _ = os.path.splitext(basename) | |
| #print(filename_without_extension) | |
| return filename_without_extension | |
| def get_file_path_end_with_ext(file_path): | |
| match = re.search(r'(.*[\/\\])?(.+)$', file_path) | |
| filename_end = match.group(2) if match else '' | |
| return filename_end | |
| def dummy_function(in_colnames): | |
| """ | |
| A dummy function that exists just so that dropdown updates work correctly. | |
| """ | |
| return None | |