Initial commit

.gitignore

@@ -0,0 +1,16 @@
+*.csv
+*.pyc
+*.cpython-311.pyc
+*.cpython-310.pyc
+*.bat
+*.json
+*.xlsx
+*.parquet
+*.json
+*.bat
+*.pkl
+*.spec
+*.ipynb
+build/*
+dist/*
+__pycache__/*

README.md

@@ -1,2 +1,13 @@
-# data_text_search
-Search for key terms in tabular text data
+---
+title: Data text search
+emoji: 🚀
+colorFrom: green
+colorTo: indigo
+sdk: gradio
+sdk_version: 3.50.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+---
+
+Adaptation of fast_bm25 (https://github.com/Inspirateur/Fast-BM25) to search over your data.

README_additions.md

@@ -0,0 +1 @@
+Adaptation of fast_bm25 (https://github.com/Inspirateur/Fast-BM25) to your data.

data_text_search.py

@@ -0,0 +1,347 @@
+from search_funcs.fast_bm25 import BM25
+from search_funcs.clean_funcs import initial_clean, get_lemma_tokens#, stem_sentence
+from nltk import word_tokenize
+
+import gradio as gr
+import pandas as pd
+import os
+
+def prepare_input_data(in_file, text_column, clean="No", progress=gr.Progress()):
+
+    filename = in_file.name
+    # Import data
+
+    df = read_file(filename)
+
+    #df = pd.read_parquet(file_in.name)
+    df_list = list(df[text_column].astype(str))
+    #df_list = df
+
+    if clean == "Yes":
+        df_list_clean = initial_clean(df_list)
+
+        # Save to file if you have cleaned the data
+        out_file_name = save_prepared_data(in_file, df_list_clean, df, text_column)
+
+        #corpus = [word_tokenize(doc.lower()) for doc in df_list_clean]
+        corpus = [word_tokenize(doc.lower()) for doc in progress.tqdm(df_list_clean, desc = "Tokenising text", unit = "rows")]
+        
+
+
+    else: 
+        #corpus = [word_tokenize(doc.lower()) for doc in df_list]
+        corpus = [word_tokenize(doc.lower()) for doc in progress.tqdm(df_list, desc = "Tokenising text", unit = "rows")]
+        out_file_name = None
+
+    
+
+    print("Finished data clean")
+
+    if len(df_list) >= 20:
+        message = "Data loaded"
+    else:
+        message = "Data loaded. Warning: dataset may be too short to get consistent search results."
+    
+    return corpus, message, df, out_file_name
+
+def get_file_path_end(file_path):
+    # First, get the basename of the file (e.g., "example.txt" from "/path/to/example.txt")
+    basename = os.path.basename(file_path)
+    
+    # Then, split the basename and its extension and return only the basename without the extension
+    filename_without_extension, _ = os.path.splitext(basename)
+
+    print(filename_without_extension)
+    
+    return filename_without_extension
+
+def save_prepared_data(in_file, prepared_text_list, in_df, in_column):
+
+    # Check if the list and the dataframe have the same length
+    if len(prepared_text_list) != len(in_df):
+        raise ValueError("The length of 'prepared_text_list' and 'in_df' must match.")
+    
+    file_end = ".parquet"
+
+    file_name = get_file_path_end(in_file.name) + "_cleaned" + file_end
+
+    prepared_text_df = pd.DataFrame(data={in_column + "_cleaned":prepared_text_list})
+
+    # Drop original column from input file to reduce file size
+    in_df = in_df.drop(in_column, axis = 1)
+
+    prepared_df = pd.concat([in_df, prepared_text_df], axis = 1)
+
+    if file_end == ".csv":
+        prepared_df.to_csv(file_name)
+    elif file_end == ".parquet":
+        prepared_df.to_parquet(file_name)
+    else: file_name = None
+    
+
+    return file_name
+
+def prepare_bm25(corpus, k1=1.5, b = 0.75, alpha=-5):
+    #bm25.save("saved_df_bm25")
+    #bm25 = BM25.load(re.sub(r'\.pkl$', '', file_in.name))
+
+    print("Preparing BM25 corpus")
+
+    global bm25
+    bm25 = BM25(corpus, k1=k1, b=b, alpha=alpha)
+
+    message = "Search parameters loaded."
+
+    print(message)
+
+    return message
+
+def convert_query_to_tokens(free_text_query, clean="No"):
+    '''
+    Split open text query into tokens and then lemmatise to get the core of the word
+    '''  
+
+    if clean=="Yes":
+        split_query = word_tokenize(free_text_query.lower())
+        out_query = get_lemma_tokens(split_query)
+        #out_query = stem_sentence(free_text_query)
+    else: 
+        split_query = word_tokenize(free_text_query.lower())
+        out_query = split_query
+
+    return out_query
+
+def bm25_search(free_text_query, in_no_search_results, original_data, text_column, clean = "No", in_join_file = None, in_join_column = "", search_df_join_column = ""):   
+
+    # Prepare query
+    if (clean == "Yes") | (text_column.endswith("_cleaned")):
+        token_query = convert_query_to_tokens(free_text_query, clean="Yes")
+    else:
+        token_query = convert_query_to_tokens(free_text_query, clean="No")
+
+    print(token_query)
+
+    # Perform search
+    print("Searching")
+
+    results_index, results_text, results_scores = bm25.extract_documents_and_scores(token_query, bm25.corpus, n=in_no_search_results) #bm25.corpus #original_data[text_column]
+    if not results_index:
+        return "No search results found", None, token_query
+
+    print("Search complete")
+
+    # Prepare results and export
+    joined_texts = [' '.join(inner_list) for inner_list in results_text]
+    results_df = pd.DataFrame(data={"index": results_index,
+                                    "search_text": joined_texts,
+                                    "search_score_abs": results_scores})
+    results_df['search_score_abs'] = abs(round(results_df['search_score_abs'], 2))
+    results_df_out = results_df[['index', 'search_text', 'search_score_abs']].merge(original_data,left_on="index", right_index=True, how="left")#.drop("index", axis=1)
+    
+    # Join on additional files
+    if in_join_file:
+        join_filename = in_join_file.name
+
+        # Import data
+        join_df = read_file(join_filename)
+        join_df[in_join_column] = join_df[in_join_column].astype(str).str.replace("\.0$","", regex=True)
+        results_df_out[search_df_join_column] = results_df_out[search_df_join_column].astype(str).str.replace("\.0$","", regex=True)
+
+        results_df_out = results_df_out.merge(join_df,left_on=search_df_join_column, right_on=in_join_column, how="left").drop(in_join_column, axis=1)
+    
+
+    # Reorder results by score
+    results_df_out = results_df_out.sort_values('search_score_abs', ascending=False)
+
+
+
+    # Out file
+    results_df_name = "search_result.csv"
+    results_df_out.to_csv(results_df_name, index= None)
+    results_first_text = results_df_out[text_column].iloc[0]
+
+    print("Returning results")
+
+    return results_first_text, results_df_name, token_query
+
+def detect_file_type(filename):
+    """Detect the file type based on its extension."""
+    if (filename.endswith('.csv')) | (filename.endswith('.csv.gz')) | (filename.endswith('.zip')):
+        return 'csv'
+    elif filename.endswith('.xlsx'):
+        return 'xlsx'
+    elif filename.endswith('.parquet'):
+        return 'parquet'
+    else:
+        raise ValueError("Unsupported file type.")
+
+def read_file(filename):
+    """Read the file based on its detected type."""
+    file_type = detect_file_type(filename)
+    
+    if file_type == 'csv':
+        return pd.read_csv(filename, low_memory=False).reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
+    elif file_type == 'xlsx':
+        return pd.read_excel(filename).reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
+    elif file_type == 'parquet':
+        return pd.read_parquet(filename).reset_index().drop(["index", "Unnamed: 0"], axis=1, errors="ignore")
+
+def put_columns_in_df(in_file, in_column):
+    '''
+    When file is loaded, update the column dropdown choices and change 'clean data' dropdown option to 'no'.
+    '''
+
+    new_choices = []
+    concat_choices = []
+    
+    
+    df = read_file(in_file.name)
+    new_choices = list(df.columns)
+
+    print(new_choices)
+
+    concat_choices.extend(new_choices)     
+        
+    return gr.Dropdown(choices=concat_choices), gr.Dropdown(value="No", choices = ["Yes", "No"]),\
+        gr.Dropdown(choices=concat_choices)
+
+def put_columns_in_join_df(in_file, in_column):
+    '''
+    When file is loaded, update the column dropdown choices and change 'clean data' dropdown option to 'no'.
+    '''
+
+    print("in_column")
+
+    new_choices = []
+    concat_choices = []
+    
+    
+    df = read_file(in_file.name)
+    new_choices = list(df.columns)
+
+    print(new_choices)
+
+    concat_choices.extend(new_choices)     
+        
+    return gr.Dropdown(choices=concat_choices)
+
+def dummy_function(gradio_component):
+    """
+    A dummy function that exists just so that dropdown updates work correctly.
+    """
+    return None    
+
+def display_info(info_component):
+    gr.Info(info_component)
+# %%
+# ## Gradio app - BM25 search
+block = gr.Blocks(theme = gr.themes.Base())
+
+with block:  
+
+    corpus_state = gr.State()
+    data_state = gr.State(pd.DataFrame())
+
+    in_k1_info = gr.State("""k1: Constant used for influencing the term frequency saturation. After saturation is reached, additional
+presence for the term adds a significantly less additional score. According to [1]_, experiments suggest
+that 1.2 < k1 < 2 yields reasonably good results, although the optimal value depends on factors such as
+the type of documents or queries. Information taken from https://github.com/Inspirateur/Fast-BM25""")
+    in_b_info = gr.State("""b: Constant used for influencing the effects of different document lengths relative to average document length.
+When b is bigger, lengthier documents (compared to average) have more impact on its effect. According to
+[1]_, experiments suggest that 0.5 < b < 0.8 yields reasonably good results, although the optimal value
+depends on factors such as the type of documents or queries. Information taken from https://github.com/Inspirateur/Fast-BM25""")
+    in_alpha_info = gr.State("""alpha: IDF cutoff, terms with a lower idf score than alpha will be dropped. A higher alpha will lower the accuracy of BM25 but increase performance. Information taken from https://github.com/Inspirateur/Fast-BM25""")
+    in_no_search_info = gr.State("""Search results number: Maximum number of search results that will be returned. Bear in mind that if the alpha value is greater than the minimum, common words will be removed from the dataset, and so the number of search results returned may be lower than this value.""")
+    in_clean_info = gr.State("""Clean text: Clean the input text and search query. The function will try to remove email components and tags, and then will 'stem' the words. I.e. it will remove the endings of words (e.g. smashed becomes smash) so that the search engine is looking for the common 'core' of words between the query and dataset.""")
+
+    gr.Markdown(
+    """
+    # Fast text search
+    Enter a text query below to search through a text data column and find relevant entries. Your data should contain at least 20 entries for the search to return results.
+    """)
+    
+    with gr.Tab(label="Search your data"):
+        with gr.Accordion(label = "Load in data", open=True):
+            in_corpus = gr.File(label="Upload your search data here")
+            with gr.Row():
+                in_column = gr.Dropdown(label="Enter the name of the text column in the data file to search")
+                
+                load_data_button = gr.Button(value="Load data")
+                 
+
+            with gr.Row():
+                load_finished_message = gr.Textbox(label="Load progress", scale = 2)
+
+
+        with gr.Accordion(label = "Search data", open=True):
+            with gr.Row():
+                in_query = gr.Textbox(label="Enter your search term")
+                mod_query = gr.Textbox(label="Cleaned search term (the terms that are passed to the search engine)")
+                             
+            search_button = gr.Button(value="Search text")
+
+            with gr.Row():
+                output_single_text = gr.Textbox(label="Top result")
+                output_file = gr.File(label="File output")
+            
+
+    with gr.Tab(label="Advanced options"):
+        with gr.Accordion(label="Data load / save options", open = False):
+            #with gr.Row():
+            in_clean_data = gr.Dropdown(label = "Clean text during load (remove tags, stem words). This will take some time!", value="No", choices=["Yes", "No"])
+            #save_clean_data_button = gr.Button(value = "Save loaded data to file", scale = 1)
+        with gr.Accordion(label="Search options", open = False):
+            with gr.Row():
+                in_k1 = gr.Slider(label = "k1 value", value = 1.5, minimum = 0.1, maximum = 5, step = 0.1, scale = 3)
+                in_k1_button = gr.Button(value = "k1 value info", scale = 1)
+            with gr.Row():
+                in_b = gr.Slider(label = "b value", value = 0.75, minimum = 0.1, maximum = 5, step = 0.05, scale = 3)
+                in_b_button = gr.Button(value = "b value info", scale = 1)
+            with gr.Row():
+                in_alpha = gr.Slider(label = "alpha value / IDF cutoff", value = -5, minimum = -5, maximum = 10, step = 1, scale = 3)
+                in_alpha_button = gr.Button(value = "alpha value info", scale = 1)
+            with gr.Row():
+                in_no_search_results = gr.Slider(label="Maximum number of search results to return", value = 100000, minimum=10, maximum=100000, step=10, scale = 3)
+                in_no_search_results_button = gr.Button(value = "Search results number info", scale = 1)
+            with gr.Row():
+                in_search_param_button = gr.Button(value="Load search parameters (Need to click this if you changed anything above)")
+        with gr.Accordion(label = "Join on additional dataframes to results", open = False):
+            in_join_file = gr.File(label="Upload your data to join here")
+            in_join_column = gr.Dropdown(label="Column to join in new data frame")
+            search_df_join_column = gr.Dropdown(label="Column to join in search data frame")
+
+        in_search_param_button.click(fn=prepare_bm25, inputs=[corpus_state, in_k1, in_b, in_alpha], outputs=[load_finished_message])
+                      
+    # ---
+    in_k1_button.click(display_info, inputs=in_k1_info)
+    in_b_button.click(display_info, inputs=in_b_info)
+    in_alpha_button.click(display_info, inputs=in_alpha_info)
+    in_no_search_results_button.click(display_info, inputs=in_no_search_info)
+    
+
+    in_corpus.upload(put_columns_in_df, inputs=[in_corpus, in_column], outputs=[in_column, in_clean_data, search_df_join_column])
+    in_join_file.upload(put_columns_in_join_df, inputs=[in_join_file, in_join_column], outputs=[in_join_column])
+
+    # Load in the data
+    load_data_button.click(fn=prepare_input_data, inputs=[in_corpus, in_column, in_clean_data], outputs=[corpus_state, load_finished_message, data_state, output_file]).\
+    then(fn=prepare_bm25, inputs=[corpus_state, in_k1, in_b, in_alpha], outputs=[load_finished_message]).\
+    then(fn=put_columns_in_df, inputs=[in_corpus, in_column], outputs=[in_column, in_clean_data, search_df_join_column])
+
+    #save_clean_data_button.click(fn=save_prepared_data, inputs=[in_corpus, corpus_state, data_state, in_column], outputs=[output_file])   
+    
+
+    # Search functions on click or enter
+    search_button.click(fn=bm25_search, inputs=[in_query, in_no_search_results, data_state, in_column, in_clean_data, in_join_file, in_join_column, search_df_join_column],
+                        outputs=[output_single_text, output_file, mod_query], api_name="search")
+    
+    in_query.submit(fn=bm25_search, inputs=[in_query, in_no_search_results, data_state, in_column, in_clean_data, in_join_file, in_join_column, search_df_join_column],
+                        outputs=[output_single_text, output_file, mod_query])
+    
+    # Dummy functions just to get dropdowns to work correctly with Gradio 3.50
+    in_column.change(dummy_function, in_column, None)
+    search_df_join_column.change(dummy_function, search_df_join_column, None)
+    in_join_column.change(dummy_function, in_join_column, None)
+
+block.queue().launch(debug=True)
+
+

hook-gradio.py

@@ -0,0 +1,9 @@
+from PyInstaller.utils.hooks import collect_data_files
+
+hiddenimports = [
+    'gradio',
+    # Add any other submodules that PyInstaller doesn't detect
+]
+
+# Use collect_data_files to find data files. Replace 'gradio' with the correct package name if it's different.
+datas = collect_data_files('gradio')

how_to_create_exe_dist.txt

@@ -0,0 +1,21 @@
+1. Create minimal environment to run the app in conda. E.g. 'conda create --name new_env'
+
+2. Activate the environment 'conda activate new_env'
+
+3. cd to this folder. Install packages from requirements.txt using 'pip install -r requirements.txt' 
+
+4. In file explorer, navigate to the miniconda/envs/new_env/Lib/site-packages/gradio-client/ folder
+
+5. Copy types.json from the gradio_client folder to the folder containing the data_text_search.py file
+
+6. pip install pyinstaller
+
+7. In command line, cd to this folder. Then run the following 'python -m PyInstaller --additional-hooks-dir=. --hidden-import pyarrow.vendored.version --add-data="types.json;gradio_client" --clean --onefile --clean --name DataSearchApp data_text_search.py'
+
+8. A 'dist' folder will be created with the executable inside along with all dependencies('dist\data_text_search'). 
+
+9. In file explorer, navigate to the miniconda/envs/new_env/Lib/site-packages/gradio/ folder. Copy the entire folder. Paste this into the new distributable subfolder 'dist\data_text_search\_internal'
+
+10. In 'dist\data_text_search' try double clicking on the .exe file. After a short delay, the command prompt should inform you about the ip address of the app that is now running. Copy the ip address, but do not close this window.
+
+11. In an Internet browser, navigate to the indicated IP address. The app should now be running in your browser window.

requirements.txt

@@ -0,0 +1,5 @@
+pandas
+nltk
+pyarrow
+openpyxl
+gradio==3.50.0

search_funcs/clean_funcs.py

@@ -0,0 +1,350 @@
+# ## Some functions to clean text
+
+# ### Some other suggested cleaning approaches
+#
+# #### From here: https://shravan-kuchkula.github.io/topic-modeling/#interactive-plot-showing-results-of-k-means-clustering-lda-topic-modeling-and-sentiment-analysis
+#
+# - remove_hyphens
+# - tokenize_text
+# - remove_special_characters
+# - convert to lower case
+# - remove stopwords
+# - lemmatize the token
+# - remove short tokens
+# - keep only words in wordnet
+# - I ADDED ON - creating custom stopwords list
+
+# +
+# Create a custom stop words list
+import nltk
+import re
+import string
+from nltk.stem import WordNetLemmatizer
+from nltk.stem import PorterStemmer
+from nltk.corpus import wordnet as wn
+from nltk import word_tokenize
+
+# Add calendar months onto stop words
+import calendar
+from tqdm import tqdm
+import gradio as gr
+
+stemmer = PorterStemmer()
+
+
+nltk.download('stopwords')
+nltk.download('wordnet')
+
+#nltk.download('words')
+#nltk.download('names')
+
+#nltk.corpus.words.words('en')  
+
+#from sklearn.feature_extraction import text
+# Adding common names to stopwords
+
+all_names = [x.lower() for x in list(nltk.corpus.names.words())]
+
+# Adding custom words to the stopwords
+custom_words = []
+my_stop_words = custom_words
+
+
+cal_month = (list(calendar.month_name))
+cal_month = [x.lower() for x in cal_month]
+
+# Remove blanks
+cal_month = [i for i in cal_month if i]
+#print(cal_month)
+custom_words.extend(cal_month)
+    
+#my_stop_words = frozenset(text.ENGLISH_STOP_WORDS.union(custom_words).union(all_names))
+#custom_stopwords = my_stop_words
+# -
+
+# #### Some of my cleaning functions
+'''
+# +
+# Remove all html elements from the text. Inspired by this: https://stackoverflow.com/questions/9662346/python-code-to-remove-html-tags-from-a-string
+
+def remove_email_start(text):
+  cleanr = re.compile('.*importance:|.*subject:')
+  cleantext = re.sub(cleanr, '', text)
+  return cleantext
+
+def remove_email_end(text):
+  cleanr = re.compile('kind regards.*|many thanks.*|sincerely.*')
+  cleantext = re.sub(cleanr, '', text)
+  return cleantext
+    
+def cleanhtml(text):
+  cleanr = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});|\xa0')
+  cleantext = re.sub(cleanr, '', text)
+  return cleantext
+
+## The above doesn't work when there is no > at the end of the string to match the initial <. Trying this: <[^>]+> but needs work: https://stackoverflow.com/questions/2013124/regex-matching-up-to-the-first-occurrence-of-a-character
+
+# Remove all email addresses and numbers from the text
+
+def cleanemail(text):
+  cleanr = re.compile('\S*@\S*\s?|\xa0')
+  cleantext = re.sub(cleanr, '', text)
+  return cleantext
+
+def cleannum(text):
+  cleanr = re.compile(r'[0-9]+')
+  cleantext = re.sub(cleanr, '', text)
+  return cleantext
+
+def cleanpostcode(text):
+  cleanr = re.compile(r'(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2})|((GIR ?0A{2})\b$)|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$)|(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\b$)')
+  cleantext = re.sub(cleanr, '', text)
+  return cleantext
+
+def cleanwarning(text):
+  cleanr = re.compile('caution: this email originated from outside of the organization. do not click links or open attachments unless you recognize the sender and know the content is safe.')
+  cleantext = re.sub(cleanr, '', text)
+  return cleantext
+
+
+# -
+
+def initial_clean(texts):
+    clean_texts = []
+    for text in texts:
+        text = remove_email_start(text)
+        text = remove_email_end(text)
+        text = cleanpostcode(text)
+        text = remove_hyphens(text)
+        text = cleanhtml(text)
+        text = cleanemail(text)
+        #text = cleannum(text)        
+        clean_texts.append(text)
+    return clean_texts
+'''
+# Pre-compiling the regular expressions for efficiency
+email_start_pattern = re.compile('.*importance:|.*subject:')
+email_end_pattern = re.compile('kind regards.*|many thanks.*|sincerely.*')
+html_pattern = re.compile('<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});|\xa0')
+email_pattern = re.compile('\S*@\S*\s?')
+num_pattern = re.compile(r'[0-9]+')
+postcode_pattern = re.compile(r'(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2})|((GIR ?0A{2})\b$)|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$)|(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\b$)')
+warning_pattern = re.compile('caution: this email originated from outside of the organization. do not click links or open attachments unless you recognize the sender and know the content is safe.')
+nbsp_pattern = re.compile(r'&nbsp;')
+
+def stem_sentence(sentence):
+
+    words = sentence.split()
+    stemmed_words = [stemmer.stem(word).lower().rstrip("'") for word in words]
+    return stemmed_words
+
+def stem_sentences(sentences, progress=gr.Progress()):
+        """Stem each sentence in a list of sentences."""
+        stemmed_sentences = [stem_sentence(sentence) for sentence in progress.tqdm(sentences)]
+        return stemmed_sentences
+
+
+
+def get_lemma_text(text):
+    # Tokenize the input string into words
+    tokens = word_tokenize(text)
+    
+    lemmas = []
+    for word in tokens:
+        if len(word) > 3:
+            lemma = wn.morphy(word)
+        else:
+            lemma = None
+        
+        if lemma is None:
+            lemmas.append(word)
+        else:
+            lemmas.append(lemma)
+    return lemmas
+
+def get_lemma_tokens(tokens):
+    # Tokenize the input string into words
+    
+    lemmas = []
+    for word in tokens:
+        if len(word) > 3:
+            lemma = wn.morphy(word)
+        else:
+            lemma = None
+        
+        if lemma is None:
+            lemmas.append(word)
+        else:
+            lemmas.append(lemma)
+    return lemmas
+
+def initial_clean(texts , progress=gr.Progress()):
+    clean_texts = []
+
+    i = 1
+    #progress(0, desc="Cleaning texts")
+    for text in progress.tqdm(texts, desc = "Cleaning data", unit = "rows"):
+        #print("Cleaning row: ", i)
+        text = re.sub(email_start_pattern, '', text)
+        text = re.sub(email_end_pattern, '', text)
+        text = re.sub(postcode_pattern, '', text)
+        text = remove_hyphens(text)  
+        text = re.sub(html_pattern, '', text)
+        text = re.sub(email_pattern, '', text)
+        text = re.sub(nbsp_pattern, '', text)
+        #text = re.sub(warning_pattern, '', text)
+        #text = stem_sentence(text)
+        text = get_lemma_text(text)
+        text = ' '.join(text)
+        # Uncomment the next line if you want to remove numbers as well
+        # text = re.sub(num_pattern, '', text)        
+        clean_texts.append(text)
+
+        i += 1
+    return clean_texts
+
+# Sample execution
+#sample_texts = [
+#    "Hello, this is a test email. kind regards, John",
+#    "<div>Email content here</div> many thanks, Jane",
+#   "caution: this email originated from outside of the organization. do not click links or open attachments unless you recognize the sender and know the content is safe.",
+#    "[email protected]",
+#    "Address: 1234 Elm St, AB12 3CD"
+#]
+
+#initial_clean(sample_texts)
+
+
+# +
+
+all_names = [x.lower() for x in list(nltk.corpus.names.words())]
+
+def remove_hyphens(text_text):
+    return re.sub(r'(\w+)-(\w+)-?(\w)?', r'\1 \2 \3', text_text)
+
+# tokenize text
+def tokenize_text(text_text):
+    TOKEN_PATTERN = r'\s+'
+    regex_wt = nltk.RegexpTokenizer(pattern=TOKEN_PATTERN, gaps=True)
+    word_tokens = regex_wt.tokenize(text_text)
+    return word_tokens
+
+def remove_characters_after_tokenization(tokens):
+    pattern = re.compile('[{}]'.format(re.escape(string.punctuation)))
+    filtered_tokens = filter(None, [pattern.sub('', token) for token in tokens])
+    return filtered_tokens
+
+def convert_to_lowercase(tokens):
+    return [token.lower() for token in tokens if token.isalpha()]
+
+def remove_stopwords(tokens, custom_stopwords):
+    stopword_list = nltk.corpus.stopwords.words('english')
+    stopword_list += my_stop_words
+    filtered_tokens = [token for token in tokens if token not in stopword_list]
+    return filtered_tokens
+
+def remove_names(tokens):
+    stopword_list = list(nltk.corpus.names.words())
+    stopword_list = [x.lower() for x in stopword_list]
+    filtered_tokens = [token for token in tokens if token not in stopword_list]
+    return filtered_tokens
+
+
+
+def remove_short_tokens(tokens):
+    return [token for token in tokens if len(token) > 3]
+
+def keep_only_words_in_wordnet(tokens):
+    return [token for token in tokens if wn.synsets(token)]
+
+def apply_lemmatize(tokens, wnl=WordNetLemmatizer()):
+
+    def lem_word(word):
+    
+        if len(word) > 3: out_word = wnl.lemmatize(word)
+        else: out_word = word
+
+        return out_word
+
+    return [lem_word(token) for token in tokens]
+
+
+# +
+### Do the cleaning
+
+def cleanTexttexts(texts):
+    clean_texts = []
+    for text in texts:
+        #text = remove_email_start(text)
+        #text = remove_email_end(text)
+        text = remove_hyphens(text)
+        text = cleanhtml(text)
+        text = cleanemail(text)
+        text = cleanpostcode(text)
+        text = cleannum(text)
+        #text = cleanwarning(text)
+        text_i = tokenize_text(text)
+        text_i = remove_characters_after_tokenization(text_i)
+        #text_i = remove_names(text_i)
+        text_i = convert_to_lowercase(text_i)
+        #text_i = remove_stopwords(text_i, my_stop_words)
+        text_i = get_lemma(text_i)
+        #text_i = remove_short_tokens(text_i)
+        text_i = keep_only_words_in_wordnet(text_i)
+
+        text_i = apply_lemmatize(text_i)
+        clean_texts.append(text_i)
+    return clean_texts
+
+
+# -
+
+def remove_dups_text(data_samples_ready, data_samples_clean, data_samples):
+   # Identify duplicates in the data: https://stackoverflow.com/questions/44191465/efficiently-identify-duplicates-in-large-list-500-000
+    # Only identifies the second duplicate
+
+    seen = set()
+    dupes = []
+
+    for i, doi in enumerate(data_samples_ready):
+        if doi not in seen:
+            seen.add(doi)
+        else:
+            dupes.append(i) 
+    #data_samples_ready[dupes[0:]]
+    
+    # To see a specific duplicated value you know the position of
+    #matching = [s for s in data_samples_ready if data_samples_ready[83] in s]
+    #matching
+    
+    # Remove duplicates only (keep first instance)
+    #data_samples_ready = list( dict.fromkeys(data_samples_ready) ) # This way would keep one version of the duplicates
+    
+    ### Remove all duplicates including original instance
+    
+    # Identify ALL duplicates including initial values
+    # https://stackoverflow.com/questions/11236006/identify-duplicate-values-in-a-list-in-python
+
+    from collections import defaultdict
+    D = defaultdict(list)
+    for i,item in enumerate(data_samples_ready):
+        D[item].append(i)
+    D = {k:v for k,v in D.items() if len(v)>1}
+    
+    # https://stackoverflow.com/questions/952914/how-to-make-a-flat-list-out-of-a-list-of-lists
+    L = list(D.values())
+    flat_list_dups = [item for sublist in L for item in sublist]
+
+    # https://stackoverflow.com/questions/11303225/how-to-remove-multiple-indexes-from-a-list-at-the-same-time
+    for index in sorted(flat_list_dups, reverse=True):
+        del data_samples_ready[index]
+        del data_samples_clean[index]
+        del data_samples[index]
+    
+    # Remove blanks
+    data_samples_ready = [i for i in data_samples_ready if i]
+    data_samples_clean = [i for i in data_samples_clean if i]
+    data_samples = [i for i in data_samples if i]
+    
+    return data_samples_ready, data_samples_clean, flat_list_dups, data_samples
+

search_funcs/fast_bm25.py

@@ -0,0 +1,198 @@
+import collections
+import heapq
+import math
+import pickle
+import sys
+from numpy import inf
+import gradio as gr
+
+PARAM_K1 = 1.5
+PARAM_B = 0.75
+IDF_CUTOFF = -inf
+
+# Built off https://github.com/Inspirateur/Fast-BM25
+
+class BM25:
+	"""Fast Implementation of Best Matching 25 ranking function.
+
+	Attributes
+	----------
+	t2d : <token: <doc, freq>>
+		Dictionary with terms frequencies for each document in `corpus`.
+	idf: <token, idf score>
+		Pre computed IDF score for every term.
+	doc_len : list of int
+		List of document lengths.
+	avgdl : float
+		Average length of document in `corpus`.
+	"""
+	def __init__(self, corpus, k1=PARAM_K1, b=PARAM_B, alpha=IDF_CUTOFF):
+		"""
+		Parameters
+		----------
+		corpus : list of list of str
+			Given corpus.
+		k1 : float
+			Constant used for influencing the term frequency saturation. After saturation is reached, additional
+			presence for the term adds a significantly less additional score. According to [1]_, experiments suggest
+			that 1.2 < k1 < 2 yields reasonably good results, although the optimal value depends on factors such as
+			the type of documents or queries.
+		b : float
+			Constant used for influencing the effects of different document lengths relative to average document length.
+			When b is bigger, lengthier documents (compared to average) have more impact on its effect. According to
+			[1]_, experiments suggest that 0.5 < b < 0.8 yields reasonably good results, although the optimal value
+			depends on factors such as the type of documents or queries.
+		alpha: float
+			IDF cutoff, terms with a lower idf score than alpha will be dropped. A higher alpha will lower the accuracy
+			of BM25 but increase performance
+		"""
+		self.k1 = k1
+		self.b = b
+		self.alpha = alpha
+		self.corpus = corpus
+
+		self.avgdl = 0
+		self.t2d = {}
+		self.idf = {}
+		self.doc_len = []
+		if corpus:
+			self._initialize(corpus)
+
+	@property
+	def corpus_size(self):
+		return len(self.doc_len)
+
+	def _initialize(self, corpus, progress=gr.Progress()):
+		"""Calculates frequencies of terms in documents and in corpus. Also computes inverse document frequencies."""
+		i = 0
+		for document in progress.tqdm(corpus, desc = "Preparing search index", unit = "rows"):
+			self.doc_len.append(len(document))
+
+			for word in document:
+				if word not in self.t2d:
+					self.t2d[word] = {}
+				if i not in self.t2d[word]:
+					self.t2d[word][i] = 0
+				self.t2d[word][i] += 1
+			i += 1
+
+		self.avgdl = sum(self.doc_len)/len(self.doc_len)
+		to_delete = []
+		for word, docs in self.t2d.items():
+			idf = math.log(self.corpus_size - len(docs) + 0.5) - math.log(len(docs) + 0.5)
+			# only store the idf score if it's above the threshold
+			if idf > self.alpha:
+				self.idf[word] = idf
+			else:
+				to_delete.append(word)
+		print(f"Dropping {len(to_delete)} terms")
+		for word in to_delete:
+			del self.t2d[word]
+
+		if len(self.idf) == 0:
+			print("Alpha value too high - all words removed from dataset.")
+			self.average_idf = 0
+
+		else:
+			self.average_idf = sum(self.idf.values())/len(self.idf)
+
+		if self.average_idf < 0:
+			print(
+				f'Average inverse document frequency is less than zero. Your corpus of {self.corpus_size} documents'
+				' is either too small or it does not originate from natural text. BM25 may produce'
+				' unintuitive results.',
+				file=sys.stderr
+			)
+
+	def get_top_n(self, query, documents, n=5):
+		"""
+		Retrieve the top n documents for the query.
+
+		Parameters
+		----------
+		query: list of str
+			The tokenized query
+		documents: list
+			The documents to return from
+		n: int
+			The number of documents to return
+
+		Returns
+		-------
+		list
+			The top n documents
+		"""
+		assert self.corpus_size == len(documents), "The documents given don't match the index corpus!"
+		scores = collections.defaultdict(float)
+		for token in query:
+			if token in self.t2d:
+				for index, freq in self.t2d[token].items():
+					denom_cst = self.k1 * (1 - self.b + self.b * self.doc_len[index] / self.avgdl)
+					scores[index] += self.idf[token]*freq*(self.k1 + 1)/(freq + denom_cst)
+
+		return [documents[i] for i in heapq.nlargest(n, scores.keys(), key=scores.__getitem__)]
+	
+
+	def get_top_n_with_score(self, query, documents, n=5):
+		"""
+		Retrieve the top n documents for the query along with their scores.
+
+		Parameters
+		----------
+		query: list of str
+			The tokenized query
+		documents: list
+			The documents to return from
+		n: int
+			The number of documents to return
+
+		Returns
+		-------
+		list
+			The top n documents along with their scores and row indices in the format (index, document, score)
+		"""
+		assert self.corpus_size == len(documents), "The documents given don't match the index corpus!"
+		scores = collections.defaultdict(float)
+		for token in query:
+			if token in self.t2d:
+				for index, freq in self.t2d[token].items():
+					denom_cst = self.k1 * (1 - self.b + self.b * self.doc_len[index] / self.avgdl)
+					scores[index] += self.idf[token] * freq * (self.k1 + 1) / (freq + denom_cst)
+
+		top_n_indices = heapq.nlargest(n, scores.keys(), key=scores.__getitem__)
+		return [(i, documents[i], scores[i]) for i in top_n_indices]
+	
+	def extract_documents_and_scores(self, query, documents, n=5):
+		"""
+		Extract top n documents and their scores into separate lists.
+
+		Parameters
+		----------
+		query: list of str
+			The tokenized query
+		documents: list
+			The documents to return from
+		n: int
+			The number of documents to return
+
+		Returns
+		-------
+		tuple: (list, list)
+			The first list contains the top n documents and the second list contains their scores.
+		"""
+		results = self.get_top_n_with_score(query, documents, n)
+		try:
+			indices, docs, scores = zip(*results)
+		except:
+			print("No search results returned")
+			return [], [], []
+		return list(indices), docs, list(scores)
+
+	def save(self, filename):
+		with open(f"{filename}.pkl", "wb") as fsave:
+			pickle.dump(self, fsave, protocol=pickle.HIGHEST_PROTOCOL)
+
+	@staticmethod
+	def load(filename):
+		with open(f"{filename}.pkl", "rb") as fsave:
+			return pickle.load(fsave)

search_funcs/ingest_text.py

@@ -0,0 +1,33 @@
+# %%
+import pandas as pd
+import csv
+
+# %%
+# Define your file paths
+file_dir = "../"
+extracted_file_path = file_dir + "2022_08_case_notes.txt"
+parquet_file_path = file_dir + "2022_08_case_notes.parquet"
+
+# %%
+# Read the TXT file using the csv module and convert to DataFrame
+csv.field_size_limit(1000000)  # set to a higher value
+
+data_list = []
+with open(extracted_file_path, mode='r', encoding='iso-8859-1') as file:
+    csv_reader = csv.reader(file, delimiter=',')  # Change the delimiter if needed
+    for row in csv_reader:
+        data_list.append(row)
+
+# Filter rows that have the same number of columns as the header
+header = data_list[0]
+filtered_data = [row for row in data_list if len(row) == len(header)]
+
+# Convert list of rows to DataFrame
+casenotes = pd.DataFrame(filtered_data[1:], columns=header)  # Assuming first row is header
+
+print(casenotes.head())  # Display the first few rows of the DataFrame
+
+# %%
+casenotes.to_parquet(parquet_file_path)
+
+