Init repo

.github/workflows/check_file_size.yml

@@ -0,0 +1,16 @@
+name: Check file size
+on:               # or directly `on: [push]` to run the action on every push on any branch
+  pull_request:
+    branches: [main]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check large files
+        uses: ActionsDesk/[email protected]
+        with:
+          filesizelimit: 10485760 # this is 10MB so we can sync to HF Spaces

.github/workflows/sync_hf.yml

@@ -0,0 +1,20 @@
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [main]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push https://seanpedrickcase:[email protected]/spaces/seanpedrickcase/Light-PDF-Web-QA-Chatbot main

.gitignore

@@ -0,0 +1,2 @@
+*.pyc
+*.ipynb

Dockerfile

@@ -0,0 +1,30 @@
+FROM python:3.10
+
+WORKDIR /src
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+# Switch to the "user" user
+USER user
+# Set home to the user's home directory
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH \
+    PYTHONPATH=$HOME/app \
+	PYTHONUNBUFFERED=1 \
+	GRADIO_ALLOW_FLAGGING=never \
+	GRADIO_NUM_PORTS=1 \
+	GRADIO_SERVER_NAME=0.0.0.0 \
+	GRADIO_THEME=huggingface \
+	SYSTEM=spaces
+ 
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+CMD ["python", "app.py"]

README.md

@@ -0,0 +1,13 @@
+---
+title: Light PDF web QA chatbot
+emoji: 📈
+colorFrom: yellow
+colorTo: yellow
+sdk: gradio
+sdk_version: 3.35.2
+app_file: app.py
+pinned: false
+license: apache-2.0
+---
+
+Chat with a pdf file or web page using a light language model through a Gradio interface. Quick responses even just using CPU.

app.py

@@ -0,0 +1,198 @@
+# # Load in packages
+
+# +
+import os
+from typing import TypeVar
+from langchain.embeddings import HuggingFaceEmbeddings, HuggingFaceInstructEmbeddings
+from langchain.vectorstores import FAISS
+
+
+#PandasDataFrame: type[pd.core.frame.DataFrame]
+PandasDataFrame = TypeVar('pd.core.frame.DataFrame')
+
+# Disable cuda devices if necessary
+
+#os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 
+
+#from chatfuncs.chatfuncs import *
+import chatfuncs.ingest as ing
+
+##  Load preset embeddings and vectorstore
+
+embeddings_name = "thenlper/gte-base"
+
+def load_embeddings(embeddings_name = "thenlper/gte-base"):
+
+
+    if embeddings_name == "hkunlp/instructor-large":
+        embeddings_func = HuggingFaceInstructEmbeddings(model_name=embeddings_name,
+        embed_instruction="Represent the paragraph for retrieval: ",
+        query_instruction="Represent the question for retrieving supporting documents: "
+        )
+
+    else: 
+        embeddings_func = HuggingFaceEmbeddings(model_name=embeddings_name)
+
+    global embeddings
+
+    embeddings = embeddings_func
+
+    return embeddings
+
+def get_faiss_store(faiss_vstore_folder,embeddings):
+    import zipfile
+    with zipfile.ZipFile(faiss_vstore_folder + '/' + faiss_vstore_folder + '.zip', 'r') as zip_ref:
+        zip_ref.extractall(faiss_vstore_folder)
+
+    faiss_vstore = FAISS.load_local(folder_path=faiss_vstore_folder, embeddings=embeddings)
+    os.remove(faiss_vstore_folder + "/index.faiss")
+    os.remove(faiss_vstore_folder + "/index.pkl")
+    
+    global vectorstore
+
+    vectorstore = faiss_vstore
+
+    return vectorstore
+
+import chatfuncs.chatfuncs as chatf
+
+chatf.embeddings = load_embeddings(embeddings_name)
+chatf.vectorstore = get_faiss_store(faiss_vstore_folder="faiss_embedding",embeddings=globals()["embeddings"])
+
+def docs_to_faiss_save(docs_out:PandasDataFrame, embeddings=embeddings):
+
+    print(f"> Total split documents: {len(docs_out)}")
+
+    vectorstore_func = FAISS.from_documents(documents=docs_out, embedding=embeddings)
+        
+    '''  
+    #with open("vectorstore.pkl", "wb") as f:
+        #pickle.dump(vectorstore, f) 
+    ''' 
+
+    #if Path(save_to).exists():
+    #    vectorstore_func.save_local(folder_path=save_to)
+    #else:
+    #    os.mkdir(save_to)
+    #    vectorstore_func.save_local(folder_path=save_to)
+
+    global vectorstore
+
+    vectorstore = vectorstore_func
+
+    chatf.vectorstore = vectorstore
+
+    out_message = "Document processing complete"
+
+    #print(out_message)
+    #print(f"> Saved to: {save_to}")
+
+    return out_message
+
+ # Gradio chat
+
+import gradio as gr
+
+block = gr.Blocks(css=".gradio-container {background-color: black}")
+
+with block:
+    #with gr.Row():
+    gr.Markdown("<h1><center>Lightweight PDF / web page QA bot</center></h1>")        
+    
+    gr.Markdown("By default the Lambeth Borough Plan '[Lambeth 2030 : Our Future, Our Lambeth](https://www.lambeth.gov.uk/better-fairer-lambeth/projects/lambeth-2030-our-future-our-lambeth)' is loaded. If you want to talk about another document or web page, please select below. The chatbot will not answer questions where answered can't be found on the website.\n\nIf switching topic, please click the 'New topic' button as the bot will assume follow up questions are linked to the first. Sources are shown underneath the chat area.")
+
+    with gr.Tab("Chatbot"):
+
+        with gr.Row():
+            chatbot = gr.Chatbot(height=300)
+            sources = gr.HTML(value = "Source paragraphs where I looked for answers will appear here", height=300)
+
+        with gr.Row():
+            message = gr.Textbox(
+                label="What's your question?",
+                lines=1,
+            )     
+
+
+        submit = gr.Button(value="Send message", variant="secondary", scale = 1)
+
+        examples_set = gr.Examples(label="Examples for the Lambeth Borough Plan",
+            examples=[
+                "What were the five pillars of the previous borough plan?",
+                "What is the vision statement for Lambeth?",
+                "What are the commitments for Lambeth?",
+                "What are the 2030 outcomes for Lambeth?"],
+            inputs=message,
+        )
+
+        with gr.Row():
+            current_topic = gr.Textbox(label="Current conversation topic. If you want to talk about something else, press 'New topic'", placeholder="Keywords related to the conversation topic will appear here")
+            clear = gr.Button(value="New topic", variant="secondary", scale=0)  
+
+
+    with gr.Tab("Load in a different PDF file or web page to chat"):
+        with gr.Accordion("PDF file", open = False):
+            in_pdf = gr.File(label="Upload pdf", file_count="multiple", file_types=['.pdf'])
+            load_pdf = gr.Button(value="Load in file", variant="secondary", scale=0)
+        
+        with gr.Accordion("Web page", open = False):
+            with gr.Row():
+                in_web = gr.Textbox(label="Enter webpage url")
+                in_div = gr.Textbox(label="(Advanced) Webpage div for text extraction", value="p", placeholder="p")
+            load_web = gr.Button(value="Load in webpage", variant="secondary", scale=0) 
+        
+        ingest_embed_out = gr.Textbox(label="File/webpage preparation progress")
+
+    gr.HTML(
+        "<center>Powered by Flan Alpaca and Langchain</a></center>"
+    )
+
+    ingest_text = gr.State()
+    ingest_metadata = gr.State()
+    ingest_docs = gr.State()
+
+    #embeddings_state = gr.State()
+    vectorstore_state = gr.State()
+
+    chat_history_state = gr.State()
+    instruction_prompt_out = gr.State()
+
+    #def hide_examples():
+    #    return gr.Examples.update(visible=False)
+
+    # Load in a pdf
+    load_pdf_click = load_pdf.click(ing.parse_file, inputs=[in_pdf], outputs=[ingest_text]).\
+             then(ing.text_to_docs, inputs=[ingest_text], outputs=[ingest_docs]).\
+             then(docs_to_faiss_save, inputs=[ingest_docs], outputs=ingest_embed_out)
+             #then(hide_examples)
+
+    # Load in a webpage
+    load_web_click = load_web.click(ing.parse_html, inputs=[in_web, in_div], outputs=[ingest_text, ingest_metadata]).\
+             then(ing.html_text_to_docs, inputs=[ingest_text, ingest_metadata], outputs=[ingest_docs]).\
+             then(docs_to_faiss_save, inputs=[ingest_docs], outputs=ingest_embed_out)
+             #then(hide_examples)
+
+    # Load in a webpage
+
+    # Click/enter to send message action
+    response_click = submit.click(chatf.get_history_sources_final_input_prompt, inputs=[message, chat_history_state, current_topic], outputs=[chat_history_state, sources, instruction_prompt_out], queue=False).\
+                then(chatf.turn_off_interactivity, inputs=[message, chatbot], outputs=[message, chatbot], queue=False).\
+                then(chatf.produce_streaming_answer_chatbot_hf, inputs=[chatbot, instruction_prompt_out], outputs=chatbot)
+    response_click.then(chatf.highlight_found_text, [chatbot, sources], [sources]).\
+                then(chatf.add_inputs_answer_to_history,[message, chatbot, current_topic], [chat_history_state, current_topic]).\
+                then(lambda: gr.update(interactive=True), None, [message], queue=False)
+
+    response_enter = message.submit(chatf.get_history_sources_final_input_prompt, inputs=[message, chat_history_state, current_topic], outputs=[chat_history_state, sources, instruction_prompt_out], queue=False).\
+                then(chatf.turn_off_interactivity, inputs=[message, chatbot], outputs=[message, chatbot], queue=False).\
+                then(chatf.produce_streaming_answer_chatbot_hf, [chatbot, instruction_prompt_out], chatbot)    
+    response_enter.then(chatf.highlight_found_text, [chatbot, sources], [sources]).\
+                then(chatf.add_inputs_answer_to_history,[message, chatbot, current_topic], [chat_history_state, current_topic]).\
+                then(lambda: gr.update(interactive=True), None, [message], queue=False)
+    
+    # Clear box
+    clear.click(chatf.clear_chat, inputs=[chat_history_state, sources, message, current_topic], outputs=[chat_history_state, sources, message, current_topic])
+    clear.click(lambda: None, None, chatbot, queue=False)
+
+block.queue(concurrency_count=1).launch(debug=True)
+# -
+

chatfuncs/.ipynb_checkpoints/chatfuncs-checkpoint.py

@@ -0,0 +1,553 @@
+# ---
+# jupyter:
+#   jupytext:
+#     formats: ipynb,py:light
+#     text_representation:
+#       extension: .py
+#       format_name: light
+#       format_version: '1.5'
+#       jupytext_version: 1.14.6
+#   kernelspec:
+#     display_name: Python 3 (ipykernel)
+#     language: python
+#     name: python3
+# ---
+
+# +
+import os
+import datetime
+from typing import Dict, List, Tuple
+from itertools import compress
+import pandas as pd
+
+from langchain import PromptTemplate
+from langchain.chains import LLMChain
+from langchain.chains.base import Chain
+from langchain.chains.combine_documents.base import BaseCombineDocumentsChain
+from langchain.embeddings import HuggingFaceEmbeddings, HuggingFaceInstructEmbeddings
+from langchain.chains.qa_with_sources import load_qa_with_sources_chain
+from langchain.prompts import PromptTemplate
+from langchain.retrievers import TFIDFRetriever, SVMRetriever
+from langchain.vectorstores import FAISS
+from langchain.llms import HuggingFacePipeline
+
+from pydantic import BaseModel
+
+import nltk
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize
+
+import torch
+#from transformers import pipeline
+from optimum.pipelines import pipeline
+from transformers import AutoTokenizer, TextStreamer, AutoModelForSeq2SeqLM, TextIteratorStreamer
+from threading import Thread
+
+import gradio as gr
+
+
+# -
+
+# # Pre-load stopwords, vectorstore, models
+
+# +
+def get_faiss_store(faiss_vstore_folder,embeddings):
+    import zipfile
+    with zipfile.ZipFile(faiss_vstore_folder + '/faiss_lambeth_census_embedding.zip', 'r') as zip_ref:
+        zip_ref.extractall(faiss_vstore_folder)
+
+    faiss_vstore = FAISS.load_local(folder_path=faiss_vstore_folder, embeddings=embeddings)
+    os.remove(faiss_vstore_folder + "/index.faiss")
+    os.remove(faiss_vstore_folder + "/index.pkl")
+    
+    return faiss_vstore
+
+#def set_hf_api_key(api_key, chain_agent):
+    #if api_key:
+       #os.environ["HUGGINGFACEHUB_API_TOKEN"] = api_key
+        #vectorstore = get_faiss_store(faiss_vstore_folder="faiss_lambeth_census_embedding.zip",embeddings=embeddings)
+        #qa_chain = create_prompt_templates(vectorstore)
+        #print(qa_chain)
+        #os.environ["HUGGINGFACEHUB_API_TOKEN"] = ""
+        #return qa_chain
+
+
+# -
+
+def create_hf_model(model_name = "declare-lab/flan-alpaca-large"):
+
+    model_id = model_name
+    torch_device = "cuda" if torch.cuda.is_available() else "cpu"
+    print("Running on device:", torch_device)
+    print("CPU threads:", torch.get_num_threads())
+    
+
+
+    if torch_device == "cuda":
+        model = AutoModelForSeq2SeqLM.from_pretrained(model_id, load_in_8bit=True, device_map="auto")
+    else:
+        #torch.set_num_threads(8)
+        model = AutoModelForSeq2SeqLM.from_pretrained(model_id)
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+    
+    return model, tokenizer, torch_device
+
+# +
+# Add some stopwords to nltk default
+
+nltk.download('stopwords')
+stopwords = nltk.corpus.stopwords.words('english')
+#print(stopwords.words('english'))
+newStopWords = ['what','how', 'when', 'which', 'who', 'change', 'changed', 'do', 'did', 'increase', 'decrease', 'increased',
+                'decreased', 'proportion', 'percentage', 'report', 'reporting','say', 'said']
+stopwords.extend(newStopWords)
+# -
+
+# Embeddings
+#model_name = "sentence-transformers/all-MiniLM-L6-v2"
+#embeddings = HuggingFaceEmbeddings(model_name=model_name)
+embed_model_name = "hkunlp/instructor-large"
+embeddings = HuggingFaceInstructEmbeddings(model_name=embed_model_name)
+vectorstore = get_faiss_store(faiss_vstore_folder="faiss_lambeth_census_embedding",embeddings=embeddings)
+
+# +
+# Models
+
+#checkpoint = 'declare-lab/flan-alpaca-base' # Flan Alpaca Base incorrectly interprets text based on input (e.g. if you use words like increase or decrease in the question it will respond falsely often). Flan Alpaca Large is much more consistent
+checkpoint = 'declare-lab/flan-alpaca-large'
+
+model, tokenizer, torch_device = create_hf_model(model_name = checkpoint)
+
+
+# Look at this for streaming text with huggingface and langchain (last example): https://github.com/hwchase17/langchain/issues/2918
+
+streamer = TextStreamer(tokenizer, skip_prompt=True)
+
+pipe = pipeline('text2text-generation', 
+                 model = checkpoint,
+#                tokenizer = tokenizer,
+                 max_length=512, 
+                 #do_sample=True,
+                 temperature=0.000001,
+                 #top_p=0.95,
+                 #repetition_penalty=1.15,
+                 accelerator="bettertransformer",
+                 streamer=streamer
+                )
+
+checkpoint_keywords = 'ml6team/keyphrase-generation-t5-small-inspec'
+
+keyword_model = pipeline('text2text-generation', 
+                 model = checkpoint_keywords,
+                 accelerator="bettertransformer"
+                )
+
+
+# -
+
+# # Chat history
+
+def clear_chat(chat_history_state, sources, chat_message):
+    chat_history_state = []
+    sources = ''
+    chat_message = ''
+    return chat_history_state, sources, chat_message
+
+
+def _get_chat_history(chat_history: List[Tuple[str, str]]): # Limit to last 3 interactions only
+    max_chat_length = 3
+
+    if len(chat_history) > max_chat_length:
+        chat_history = chat_history[-max_chat_length:]
+        
+    print(chat_history)
+
+    first_q = ""
+    for human_s, ai_s in chat_history:
+        first_q = human_s
+        break
+
+    conversation = ""
+    for human_s, ai_s in chat_history:
+        human = f"Human: " + human_s
+        ai = f"Assistant: " + ai_s
+        conversation += "\n" + "\n".join([human, ai])
+
+    return conversation, first_q
+
+
+def adapt_q_from_chat_history(keyword_model, new_question_keywords, question, chat_history):
+        t5_small_keyphrase = HuggingFacePipeline(pipeline=keyword_model)
+        memory_llm = t5_small_keyphrase#flan_alpaca#flan_t5_xxl
+        new_q_memory_llm = t5_small_keyphrase#flan_alpaca#flan_t5_xxl
+    
+
+        memory_prompt = PromptTemplate(
+            template = "{chat_history_first_q}",
+            input_variables=["chat_history_first_q"]
+        )
+            #template = "Extract the names of people, things, or places from the following text: {chat_history}",#\n    Original question: {question}\n  New list:",
+            #template = "Extract keywords, and the names of people or places from the following text: {chat_history}",#\n    Original question: {question}\n  New list:",
+            #\n    Original question: {question}\n  New list:",
+        
+
+            #example_prompt=_eg_prompt,
+            #input_variables=["question", "chat_history"]
+            #input_variables=["chat_history"]
+        
+        memory_extractor = LLMChain(llm=memory_llm, prompt=memory_prompt)
+       
+        #new_question_keywords = #remove_stopwords(question)
+
+        print("new_question_keywords:")
+        print(new_question_keywords)
+
+        chat_history_str, chat_history_first_q = _get_chat_history(chat_history)
+        if chat_history_str:
+            
+            extracted_memory = memory_extractor.run(
+                chat_history_first_q=chat_history_first_q # question=question, chat_history=chat_history_str, 
+            )
+            
+            new_question_kworded = extracted_memory + " " + new_question_keywords
+            new_question = extracted_memory + " " + question
+            
+        else:
+            new_question = question
+            new_question_kworded = new_question_keywords
+            
+        return new_question, new_question_kworded
+
+
+# # Prompt creation
+
+def remove_q_stopwords(question):
+    # Prepare question by removing keywords
+    text = question.lower()
+    text_tokens = word_tokenize(text)
+    tokens_without_sw = [word for word in text_tokens if not word in stopwords]
+    new_question_keywords = ' '.join(tokens_without_sw)
+    return new_question_keywords, question
+
+
+def create_final_prompt(inputs: Dict[str, str], vectorstore, instruction_prompt, content_prompt):
+        
+        question =  inputs["question"]
+        chat_history = inputs["chat_history"]
+        
+        new_question_keywords, question = remove_q_stopwords(question)
+
+        new_question, new_question_kworded = adapt_q_from_chat_history(keyword_model, new_question_keywords, question, chat_history)
+        
+
+        print("The question passed to the vector search is:")
+        print(new_question_kworded)
+        
+        docs_keep_as_doc, docs_content, docs_url = find_relevant_passages(new_question_kworded, embeddings, k_val = 3, out_passages = 2, vec_score_cut_off = 1.3, vec_weight = 1, tfidf_weight = 0.5, svm_weight = 1)
+
+        if docs_keep_as_doc == []:
+            {"answer": "I'm sorry, I couldn't find a relevant answer to this question.", "sources":"I'm sorry, I couldn't find a relevant source for this question."}
+        
+        #new_inputs = inputs.copy()
+        #new_inputs["question"] = new_question
+        #new_inputs["chat_history"] = chat_history_str
+        
+        string_docs_content = '\n\n\n'.join(docs_content)
+        
+        #print("The draft instruction prompt is:")
+        #print(instruction_prompt)
+        
+        instruction_prompt_out = instruction_prompt.format(question=new_question, summaries=string_docs_content)
+        #print("The final instruction prompt:")
+        #print(instruction_prompt_out)
+        
+                
+        return instruction_prompt_out, string_docs_content
+
+
+# +
+def create_prompt_templates():    
+  
+    #EXAMPLE_PROMPT = PromptTemplate(
+    #    template="\nCONTENT:\n\n{page_content}\n\nSOURCE: {source}\n\n",
+    #    input_variables=["page_content", "source"],
+    #)
+
+    CONTENT_PROMPT = PromptTemplate(
+        template="{page_content}\n\n",#\n\nSOURCE: {source}\n\n",
+        input_variables=["page_content"]
+    )
+
+
+# The main prompt:
+
+    #main_prompt_template = """
+    #Answer the question using the CONTENT below:  
+
+    #CONTENT: {summaries}
+    
+    #QUESTION: {question}
+
+    #ANSWER: """
+
+    instruction_prompt_template = """
+    {summaries}
+    
+    QUESTION: {question}
+    
+    Quote relevant text above."""
+
+   
+    INSTRUCTION_PROMPT=PromptTemplate(template=instruction_prompt_template, input_variables=['question', 'summaries'])
+    
+    return INSTRUCTION_PROMPT, CONTENT_PROMPT
+
+
+# -
+
+def get_history_sources_final_input_prompt(user_input, history):
+    
+    #if chain_agent is None:
+    #    history.append((user_input, "Please click the button to submit the Huggingface API key before using the chatbot (top right)"))
+    #    return history, history, "", ""
+    print("\n==== date/time: " + str(datetime.datetime.now()) + " ====")
+    print("User input: " + user_input)
+    
+    history = history or []
+    
+
+    
+    # Create instruction prompt
+    instruction_prompt, content_prompt = create_prompt_templates()
+    instruction_prompt_out, string_docs_content =\
+                create_final_prompt({"question": user_input, "chat_history": history}, vectorstore,
+                                    instruction_prompt, content_prompt)
+    
+    sources_txt =  string_docs_content
+    
+    #print('sources_txt:')
+    #print(sources_txt)
+    
+    history.append(user_input)
+    
+    print("Output history is:")
+    print(history)
+
+    print("The output prompt is:")
+    print(instruction_prompt_out)
+    
+    return history, sources_txt, instruction_prompt_out
+
+
+# # Chat functions
+
+def produce_streaming_answer_chatbot(history, full_prompt): 
+    
+    print("The question is: ")
+    print(full_prompt)
+    
+    # Get the model and tokenizer, and tokenize the user text.
+    model_inputs = tokenizer(text=full_prompt, return_tensors="pt").to(torch_device)
+
+    # Start generation on a separate thread, so that we don't block the UI. The text is pulled from the streamer
+    # in the main thread. Adds timeout to the streamer to handle exceptions in the generation thread.
+    streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
+    generate_kwargs = dict(
+        model_inputs,
+        streamer=streamer,
+        max_new_tokens=512,
+        do_sample=True,
+        #top_p=top_p,
+        temperature=float(0.00001)#,
+        #top_k=top_k
+    )
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
+
+    # Pull the generated text from the streamer, and update the model output.
+    
+    history[-1][1] = ""
+    for new_text in streamer:
+        history[-1][1] += new_text
+        yield history
+
+
+def user(user_message, history):
+        return gr.update(value="", interactive=False), history + [[user_message, None]]
+
+
+def add_inputs_answer_to_history(user_message, history):
+    #history.append((user_message, [-1]))
+    
+    print("History after appending is:")
+    print(history)
+    
+    
+    return history
+
+
+# # Vector / hybrid search
+
+def find_relevant_passages(new_question_kworded, embeddings, k_val, out_passages, vec_score_cut_off, vec_weight, tfidf_weight, svm_weight, vectorstore=vectorstore):
+
+            docs = vectorstore.similarity_search_with_score(new_question_kworded, k=k_val)
+            #docs = self.vstore.similarity_search_with_score(new_question_kworded, k=k_val)
+
+            # Keep only documents with a certain score
+            #docs_orig = [x[0] for x in docs]
+            docs_scores = [x[1] for x in docs]
+
+            # Only keep sources that are sufficiently relevant (i.e. similarity search score below threshold below)
+            score_more_limit = pd.Series(docs_scores) < vec_score_cut_off
+            docs_keep = list(compress(docs, score_more_limit))
+
+            if docs_keep == []:
+                docs_keep_as_doc = []
+                docs_content = []
+                docs_url = []
+                return docs_keep_as_doc, docs_content, docs_url
+            
+            
+
+            docs_keep_as_doc = [x[0] for x in docs_keep]
+            docs_keep_length = len(docs_keep_as_doc)
+
+            #print('docs_keep:')
+            #print(docs_keep)
+
+            vec_rank = [*range(1, docs_keep_length+1)]
+            vec_score = [(docs_keep_length/x)*vec_weight for x in vec_rank]
+
+            #print("vec_rank")
+            #print(vec_rank)
+
+            #print("vec_score")
+            #print(vec_score)
+
+        
+
+            # 2nd level check on retrieved docs with TFIDF
+            content_keep=[]
+            for item in docs_keep:
+                content_keep.append(item[0].page_content)
+
+            tfidf_retriever = TFIDFRetriever.from_texts(content_keep, k = k_val)
+            tfidf_result = tfidf_retriever.get_relevant_documents(new_question_kworded)
+
+            #print("TDIDF retriever result:")
+            #print(tfidf_result)
+
+            tfidf_rank=[]
+            tfidf_score = []
+
+            for vec_item in docs_keep:
+                x = 0
+                for tfidf_item in tfidf_result:
+                    x = x + 1
+                    if tfidf_item.page_content == vec_item[0].page_content:
+                        tfidf_rank.append(x)
+                        tfidf_score.append((docs_keep_length/x)*tfidf_weight)
+
+            #print("tfidf_rank:")
+            #print(tfidf_rank)
+            #print("tfidf_score:")
+            #print(tfidf_score)
+
+
+            # 3rd level check on retrieved docs with SVM retriever
+            svm_retriever = SVMRetriever.from_texts(content_keep, embeddings, k = k_val)
+            svm_result = svm_retriever.get_relevant_documents(new_question_kworded)
+
+            #print("SVM retriever result:")
+            #print(svm_result)
+         
+            svm_rank=[]
+            svm_score = []
+
+            for vec_item in docs_keep:
+                x = 0
+                for svm_item in svm_result:
+                    x = x + 1
+                    if svm_item.page_content == vec_item[0].page_content:
+                        svm_rank.append(x)
+                        svm_score.append((docs_keep_length/x)*svm_weight)
+
+            #print("svm_score:")
+            #print(svm_score)
+
+        
+            ## Calculate final score based on three ranking methods
+            final_score = [a  + b + c for a, b, c in zip(vec_score, tfidf_score, svm_score)]
+            final_rank = [sorted(final_score, reverse=True).index(x)+1 for x in final_score]
+
+            #print("Final score:")
+            #print(final_score)
+            #print("final rank:")
+            #print(final_rank)
+
+            best_rank_index_pos = []
+
+            for x in range(1,out_passages+1):
+                try:
+                    best_rank_index_pos.append(final_rank.index(x))
+                except IndexError: # catch the error
+                    pass
+
+            # Adjust best_rank_index_pos to 
+        
+            #print("Best rank positions in original vector search list:")
+            #print(best_rank_index_pos)
+
+            best_rank_pos_series = pd.Series(best_rank_index_pos)
+            #docs_keep_out = list(compress(docs_keep, best_rank_pos_series))
+
+            #print("docs_keep:")
+            #print(docs_keep)
+
+            docs_keep_out = [docs_keep[i] for i in best_rank_index_pos]
+        
+
+            #docs_keep = [(docs_keep[best_rank_pos])]
+            # Keep only 'best' options
+            docs_keep_as_doc = [x[0] for x in docs_keep_out]# [docs_keep_as_doc_filt[0]]#[x[0] for x in docs_keep_as_doc_filt] #docs_keep_as_doc_filt[0]#
+        
+            #print("docs_keep_out:")
+            #print(docs_keep_out)
+
+            # Extract content and metadata from 'winning' passages.
+
+            content=[]
+            meta_url=[]
+            score=[]
+
+            for item in docs_keep_out:
+                content.append(item[0].page_content)
+                meta_url.append(item[0].metadata['source'])
+                score.append(item[1])       
+
+            # Create df from 'winning' passages
+
+            doc_df = pd.DataFrame(list(zip(content, meta_url, score)),
+               columns =['page_content', 'meta_url', 'score'])#.iloc[[0, 1]]
+        
+            #print("docs_keep_as_doc: ")
+            #print(docs_keep_as_doc)
+
+            #print("doc_df")
+            #print(doc_df)
+
+            docs_content = doc_df['page_content'].astype(str)
+            docs_url = "https://" + doc_df['meta_url']
+        
+            #print("Docs meta url is: ")
+            #print(docs_meta_url)
+
+            #print("Docs content is: ")
+            #print(docs_content)
+
+            #docs_url = [d['source'] for d in docs_meta]
+            #print(docs_url)
+            
+            
+
+            return docs_keep_as_doc, docs_content, docs_url

chatfuncs/.ipynb_checkpoints/ingest-checkpoint.py

@@ -0,0 +1,509 @@
+# ---
+# jupyter:
+#   jupytext:
+#     formats: ipynb,py:light
+#     text_representation:
+#       extension: .py
+#       format_name: light
+#       format_version: '1.5'
+#       jupytext_version: 1.14.6
+#   kernelspec:
+#     display_name: Python 3 (ipykernel)
+#     language: python
+#     name: python3
+# ---
+
+# # Ingest website to FAISS
+
+# ## Install/ import stuff we need
+
+import os
+from pathlib import Path
+import re
+import requests
+import pandas as pd
+import dateutil.parser
+from typing import TypeVar, List
+
+from langchain.embeddings import HuggingFaceInstructEmbeddings, HuggingFaceEmbeddings
+from langchain.vectorstores.faiss import FAISS
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.docstore.document import Document
+from langchain.document_loaders import PyPDFLoader
+
+import magic
+from bs4 import BeautifulSoup
+from docx import Document as Doc
+from pypdf import PdfReader
+from docx import Document
+
+PandasDataFrame = TypeVar('pd.core.frame.DataFrame')
+# -
+
+split_strat = [".", "!", "?", "\n\n", "\n", ",", " ", ""]
+chunk_size = 1000
+chunk_overlap = 200
+
+## Overarching ingest function:
+
+
+def determine_file_type(file_path):
+    """
+    Determine the MIME type of the given file using the magic library.
+    
+    Parameters:
+        file_path (str): Path to the file.
+    
+    Returns:
+        str: MIME type of the file.
+    """
+    return magic.from_file(file_path, mime=True)
+
+def parse_pdf(file) -> List[str]:
+
+    """
+    Extract text from a PDF file.
+    
+    Parameters:
+        file_path (str): Path to the PDF file.
+    
+    Returns:
+        List[str]: Extracted text from the PDF.
+    """
+    
+    output = []
+    for i in range(0,len(file)):
+        print(file[i].name)
+        pdf = PdfReader(file[i].name) #[i]
+        for page in pdf.pages:
+            text = page.extract_text()
+            # Merge hyphenated words
+            text = re.sub(r"(\w+)-\n(\w+)", r"\1\2", text)
+            # Fix newlines in the middle of sentences
+            text = re.sub(r"(?<!\n\s)\n(?!\s\n)", " ", text.strip())
+            # Remove multiple newlines
+            text = re.sub(r"\n\s*\n", "\n\n", text)
+            output.append(text)
+    return output
+
+
+def parse_docx(file_path):
+    """
+    Reads the content of a .docx file and returns it as a string.
+
+    Parameters:
+    - file_path (str): Path to the .docx file.
+
+    Returns:
+    - str: Content of the .docx file.
+    """
+    doc = Doc(file_path)
+    full_text = []
+    for para in doc.paragraphs:
+        full_text.append(para.text)
+    return '\n'.join(full_text)
+
+
+def parse_txt(file_path):
+    """
+    Read text from a TXT or HTML file.
+    
+    Parameters:
+        file_path (str): Path to the TXT or HTML file.
+    
+    Returns:
+        str: Text content of the file.
+    """
+    with open(file_path, 'r', encoding="utf-8") as file:
+        return file.read()
+
+
+
+def parse_file(file_paths):
+    """
+    Accepts a list of file paths, determines each file's type, 
+    and passes it to the relevant parsing function.
+    
+    Parameters:
+        file_paths (list): List of file paths.
+    
+    Returns:
+        dict: A dictionary with file paths as keys and their parsed content (or error message) as values.
+    """
+    if not isinstance(file_paths, list):
+        raise ValueError("Expected a list of file paths.")
+    
+    mime_type_to_parser = {
+        'application/pdf': parse_pdf,
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document': parse_docx,
+        'text/plain': parse_txt,
+        'text/html': parse_html
+    }
+    
+    parsed_contents = {}
+
+    for file_path in file_paths:
+        mime_type = determine_file_type(file_path)
+        if mime_type in mime_type_to_parser:
+            parsed_contents[file_path] = mime_type_to_parser[mime_type](file_path)
+        else:
+            parsed_contents[file_path] = f"Unsupported file type: {mime_type}"
+    
+    return parsed_contents
+
+
+
+
+def parse_html(page_url, div_filter="p"):
+    """
+    Determine if the source is a web URL or a local HTML file, extract the content based on the div of choice. Also tries to extract dates (WIP)
+
+    Parameters:
+        page_url (str): The web URL or local file path.
+
+    Returns:
+        str: Extracted content.
+    """
+
+    def is_web_url(s):
+    """
+    Check if the input string is a web URL.
+    """
+    return s.startswith("http://") or s.startswith("https://")
+
+    def is_local_html_file(s):
+    """
+    Check if the input string is a path to a local HTML file.
+    """
+    return (s.endswith(".html") or s.endswith(".htm")) and os.path.isfile(s)
+
+    def extract_text_from_source(source):
+    """
+    Determine if the source is a web URL or a local HTML file, 
+    and then extract its content accordingly.
+
+    Parameters:
+        source (str): The web URL or local file path.
+
+    Returns:
+        str: Extracted content.
+    """
+        if is_web_url(source):
+            response = requests.get(source)
+            response.raise_for_status()  # Raise an HTTPError for bad responses
+            return response.text
+        elif is_local_html_file(source):
+            with open(source, 'r', encoding='utf-8') as file:
+                return file.read()
+        else:
+            raise ValueError("Input is neither a valid web URL nor a local HTML file path.")
+
+    def clean_html_data(data, date_filter="", div_filt="p"):
+    """
+    Extracts and cleans data from HTML content.
+
+    Parameters:
+        data (str): HTML content to be parsed.
+        date_filter (str, optional): Date string to filter results. If set, only content with a date greater than this will be returned.
+        div_filt (str, optional): HTML tag to search for text content. Defaults to "p".
+
+    Returns:
+        tuple: Contains extracted text and date as strings. Returns empty strings if not found.
+    """
+    
+        soup = BeautifulSoup(data, 'html.parser')
+
+        # Function to exclude div with id "bar"
+        def exclude_div_with_id_bar(tag):
+            return tag.has_attr('id') and tag['id'] == 'related-links'
+
+        text_elements = soup.find_all(div_filt)
+        date_elements = soup.find_all(div_filt, {"class": "page-neutral-intro__meta"})
+    
+        # Extract date
+        date_out = ""
+        if date_elements:
+            date_out = re.search(">(.*?)<", str(date_elements[0])).group(1)
+            date_dt = dateutil.parser.parse(date_out)
+
+            if date_filter:
+                date_filter_dt = dateutil.parser.parse(date_filter)
+                if date_dt < date_filter_dt:
+                    return '', date_out
+
+        # Extract text
+        text_out_final = ""
+        if text_elements:
+            text_out_final = '\n'.join(paragraph.text for paragraph in text_elements)
+        else:
+            print(f"No elements found with tag '{div_filt}'. No text returned.")
+    
+        return text_out_final, date_out
+    
+
+    #page_url = "https://pypi.org/project/InstructorEmbedding/" #'https://www.ons.gov.uk/visualisations/censusareachanges/E09000022/index.html'
+
+    html_text = extract_text_from_source(page_url)
+    #print(page.text)
+
+    texts = []
+    metadatas = []
+
+    clean_text, date = clean_html_data(html_text, date_filter="", div_filt=div_filter)
+    texts.append(clean_text)
+    metadatas.append({"source": page_url, "date":str(date)})
+
+    return texts, metadatas
+
+
+# +
+# Convert parsed text to docs
+# -
+
+def text_to_docs(text_dict: dict, chunk_size: int = chunk_size) -> List[Document]:
+    """
+    Converts the output of parse_file (a dictionary of file paths to content)
+    to a list of Documents with metadata.
+    """
+    
+    doc_chunks = []
+
+    for file_path, content in text_dict.items():
+        ext = os.path.splitext(file_path)[1].lower()
+
+        # Depending on the file extension, handle the content
+        if ext == '.pdf':
+            docs = pdf_text_to_docs(content, chunk_size)
+        elif ext in ['.html', '.htm', '.txt', '.docx']:
+            # Assuming you want to process HTML similarly to PDF in this context
+            docs = html_text_to_docs(content, chunk_size)
+        else:
+            print(f"Unsupported file type {ext} for {file_path}. Skipping.")
+            continue
+
+        # Add filename as metadata
+        for doc in docs:
+            doc.metadata["file"] = file_path
+        
+        doc_chunks.extend(docs)
+
+    return doc_chunks
+
+
+
+def pdf_text_to_docs(text: str, chunk_size: int = chunk_size) -> List[Document]:
+    """Converts a string or list of strings to a list of Documents
+    with metadata."""
+    if isinstance(text, str):
+        # Take a single string as one page
+        text = [text]
+        
+    page_docs = [Document(page_content=page) for page in text]
+
+    # Add page numbers as metadata
+    for i, doc in enumerate(page_docs):
+        doc.metadata["page"] = i + 1
+
+    # Split pages into chunks
+    doc_chunks = []
+
+    for doc in page_docs:
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size,
+            separators=split_strat,#["\n\n", "\n", ".", "!", "?", ",", " ", ""],
+            chunk_overlap=chunk_overlap,
+        )
+        chunks = text_splitter.split_text(doc.page_content)
+
+        
+        for i, chunk in enumerate(chunks):
+            doc = Document(
+                page_content=chunk, metadata={"page": doc.metadata["page"], "chunk": i}
+            )
+            # Add sources a metadata
+            doc.metadata["page_chunk"] = f"{doc.metadata['page']}-{doc.metadata['chunk']}"
+            doc_chunks.append(doc)
+    return doc_chunks
+
+def html_text_to_docs(texts, metadatas, chunk_size:int = chunk_size):
+
+    text_splitter = RecursiveCharacterTextSplitter(
+        separators=split_strat,#["\n\n", "\n", ".", "!", "?", ",", " ", ""],
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        length_function=len
+    )
+
+    #print(texts)
+    #print(metadatas)
+
+    documents = text_splitter.create_documents(texts, metadatas=metadatas)
+
+    for i, chunk in enumerate(documents):
+        chunk.metadata["chunk"] = i + 1
+
+    return documents
+
+
+
+
+    
+
+# # Functions for working with documents after loading them back in
+
+def pull_out_data(series):
+
+    # define a lambda function to convert each string into a tuple
+    to_tuple = lambda x: eval(x)
+
+    # apply the lambda function to each element of the series
+    series_tup = series.apply(to_tuple)
+
+    series_tup_content = list(zip(*series_tup))[1]
+
+    series = pd.Series(list(series_tup_content))#.str.replace("^Main post content", "", regex=True).str.strip()
+
+    return series
+
+
+def docs_from_csv(df):
+
+    import ast
+    
+    documents = []
+    
+    page_content = pull_out_data(df["0"])
+    metadatas = pull_out_data(df["1"])
+
+    for x in range(0,len(df)):       
+        new_doc = Document(page_content=page_content[x], metadata=metadatas[x])
+        documents.append(new_doc)
+        
+    return documents
+
+
+def docs_from_lists(docs, metadatas):
+
+    documents = []
+
+    for x, doc in enumerate(docs):
+        new_doc = Document(page_content=doc, metadata=metadatas[x])
+        documents.append(new_doc)
+        
+    return documents
+
+
+def docs_elements_from_csv_save(docs_path="documents.csv"):
+
+    documents = pd.read_csv(docs_path)
+
+    docs_out = docs_from_csv(documents)
+
+    out_df = pd.DataFrame(docs_out)
+
+    docs_content = pull_out_data(out_df[0].astype(str))
+
+    docs_meta = pull_out_data(out_df[1].astype(str))
+
+    doc_sources = [d['source'] for d in docs_meta]
+
+    return out_df, docs_content, docs_meta, doc_sources
+
+
+# documents = html_text_to_docs(texts, metadatas)
+#
+# documents[0]
+#
+# pd.DataFrame(documents).to_csv("documents.csv", index=None)
+
+# ## Create embeddings and save faiss vector store to the path specified in `save_to`
+
+def load_embeddings(model_name = "hkunlp/instructor-large"):
+
+    if model_name == "hkunlp/instructor-large":
+        embeddings_func = HuggingFaceInstructEmbeddings(model_name=model_name,
+        embed_instruction="Represent the paragraph for retrieval: ",
+        query_instruction="Represent the question for retrieving supporting documents: "
+        )
+
+    else: 
+        embeddings_func = HuggingFaceEmbeddings(model_name=model_name)
+
+    global embeddings
+
+    embeddings = embeddings_func
+
+    #return embeddings_func
+
+
+def embed_faiss_save_to_zip(docs_out, save_to="faiss_lambeth_census_embedding", model_name = "hkunlp/instructor-large"):
+
+    load_embeddings(model_name=model_name)
+
+    #embeddings_fast = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+
+    print(f"> Total split documents: {len(docs_out)}")
+
+    vectorstore = FAISS.from_documents(documents=docs_out, embedding=embeddings)
+        
+
+    if Path(save_to).exists():
+        vectorstore.save_local(folder_path=save_to)
+
+    print("> DONE")
+    print(f"> Saved to: {save_to}")
+
+    ### Save as zip, then remove faiss/pkl files to allow for upload to huggingface
+
+    import shutil
+
+    shutil.make_archive(save_to, 'zip', save_to)
+
+    os.remove(save_to + "/index.faiss")
+    os.remove(save_to + "/index.pkl")
+
+    shutil.move(save_to + '.zip', save_to + "/" + save_to + '.zip')
+
+    return vectorstore
+
+
+# +
+# https://colab.research.google.com/drive/1RWqGXd2B6sPchlYVihKaBSsHy9zWRcYF#scrollTo=Q_eTIZwf4Dk2
+
+def docs_to_chroma_save(embeddings, docs_out:PandasDataFrame, save_to:str):
+    print(f"> Total split documents: {len(docs_out)}")
+    
+    vectordb = Chroma.from_documents(documents=docs_out, 
+                                 embedding=embeddings,
+                                 persist_directory=save_to)
+    
+    # persiste the db to disk
+    vectordb.persist()
+    
+    print("> DONE")
+    print(f"> Saved to: {save_to}")
+    
+    return vectordb
+
+
+# + [markdown] jp-MarkdownHeadingCollapsed=true
+# ## Similarity search on saved vectorstore
+# -
+
+def sim_search_local_saved_vec(query, k_val, save_to="faiss_lambeth_census_embedding"):
+
+    load_embeddings()
+
+    docsearch = FAISS.load_local(folder_path=save_to, embeddings=embeddings)
+
+
+    display(Markdown(question))
+
+    search = docsearch.similarity_search_with_score(query, k=k_val)
+
+    for item in search:
+        print(item[0].page_content)
+        print(f"Page: {item[0].metadata['source']}")
+        print(f"Date: {item[0].metadata['date']}")
+        print(f"Score: {item[1]}")
+        print("---")

chatfuncs/chatfuncs.py

@@ -0,0 +1,968 @@
+import re
+import datetime
+from typing import TypeVar, Dict, List, Tuple
+from itertools import compress
+import pandas as pd
+import numpy as np
+
+# Model packages
+import torch
+from threading import Thread
+from transformers import AutoTokenizer, pipeline, TextIteratorStreamer
+
+# Alternative model sources
+from gpt4all import GPT4All
+from ctransformers import AutoModelForCausalLM
+
+from dataclasses import asdict, dataclass
+
+# Langchain functions
+from langchain import PromptTemplate
+from langchain.prompts import PromptTemplate
+from langchain.vectorstores import FAISS
+from langchain.retrievers import SVMRetriever 
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.docstore.document import Document
+
+# For keyword extraction
+import nltk
+from nltk.corpus import stopwords
+from nltk.tokenize import RegexpTokenizer
+from nltk.stem import WordNetLemmatizer
+import keybert
+#from transformers.pipelines import pipeline
+
+# For Name Entity Recognition model
+from span_marker import SpanMarkerModel
+
+# For BM25 retrieval
+from gensim.corpora import Dictionary
+from gensim.models import TfidfModel, OkapiBM25Model
+from gensim.similarities import SparseMatrixSimilarity
+
+import gradio as gr
+
+torch_device = "cuda" if torch.cuda.is_available() else "cpu"
+print("Running on device:", torch_device)
+threads = torch.get_num_threads()
+print("CPU threads:", threads)
+
+PandasDataFrame = TypeVar('pd.core.frame.DataFrame')
+
+embeddings = None  # global variable setup
+vectorstore = None # global variable setup
+
+full_text = "" # Define dummy source text (full text) just to enable highlight function to load
+
+ctrans_llm = [] # Define empty list to hold CTrans LLMs for functions to run
+
+temperature: float = 0.1
+top_k: int = 3
+top_p: float = 1
+repetition_penalty: float = 1.05
+last_n_tokens: int = 64
+max_new_tokens: int = 125
+#seed: int = 42
+reset: bool = False
+stream: bool = True
+threads: int = threads
+batch_size:int = 512
+context_length:int = 2048
+gpu_layers:int = 0
+sample = False
+
+## Highlight text constants
+hlt_chunk_size = 20
+hlt_strat = [" ", ".", "!", "?", ":", "\n\n", "\n", ","]
+hlt_overlap = 0
+
+## Initialise NER model ##
+ner_model = SpanMarkerModel.from_pretrained("tomaarsen/span-marker-mbert-base-multinerd")
+
+## Initialise keyword model ##
+# Used to pull out keywords from chat history to add to user queries behind the scenes
+kw_model = pipeline("feature-extraction", model="thenlper/gte-base")
+
+## Chat models ##
+ctrans_llm = AutoModelForCausalLM.from_pretrained('TheBloke/orca_mini_3B-GGML', model_type='llama', model_file='orca-mini-3b.ggmlv3.q4_0.bin')
+#ctrans_llm = AutoModelForCausalLM.from_pretrained('TheBloke/orca_mini_3B-GGML', model_type='llama', model_file='orca-mini-3b.ggmlv3.q8_0.bin')
+#gpt4all_model = GPT4All(model_name= "orca-mini-3b.ggmlv3.q4_0.bin", model_path="models/") # "ggml-mpt-7b-chat.bin"
+
+# Huggingface chat model
+hf_checkpoint = 'declare-lab/flan-alpaca-large'
+
+def create_hf_model(model_name):
+
+    from transformers import AutoModelForSeq2SeqLM,  AutoModelForCausalLM
+
+#    model_id = model_name
+    
+    if torch_device == "cuda":
+        if "flan" in model_name:
+            model = AutoModelForSeq2SeqLM.from_pretrained(model_name, load_in_8bit=True, device_map="auto")
+        elif "mpt" in model_name:
+            model = AutoModelForCausalLM.from_pretrained(model_name, load_in_8bit=True, device_map="auto", trust_remote_code=True)
+        else:
+            model = AutoModelForCausalLM.from_pretrained(model_name, load_in_8bit=True, device_map="auto")
+    else:
+        if "flan" in model_name:
+            model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+        elif "mpt" in model_name:    
+            model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True)
+        else: 
+            model = AutoModelForCausalLM.from_pretrained(model_name)
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name, model_max_length = 2048)
+
+    return model, tokenizer, torch_device
+
+model, tokenizer, torch_device = create_hf_model(model_name = hf_checkpoint)
+
+# Vectorstore funcs
+
+def docs_to_faiss_save(docs_out:PandasDataFrame, embeddings=embeddings):
+
+    print(f"> Total split documents: {len(docs_out)}")
+
+    vectorstore_func = FAISS.from_documents(documents=docs_out, embedding=embeddings)
+        
+    '''  
+    #with open("vectorstore.pkl", "wb") as f:
+        #pickle.dump(vectorstore, f) 
+    ''' 
+
+    #if Path(save_to).exists():
+    #    vectorstore_func.save_local(folder_path=save_to)
+    #else:
+    #    os.mkdir(save_to)
+    #    vectorstore_func.save_local(folder_path=save_to)
+
+    global vectorstore
+
+    vectorstore = vectorstore_func
+
+    out_message = "Document processing complete"
+
+    #print(out_message)
+    #print(f"> Saved to: {save_to}")
+
+    return out_message
+
+# # Prompt functions
+
+def create_prompt_templates():    
+  
+    #EXAMPLE_PROMPT = PromptTemplate(
+    #    template="\nCONTENT:\n\n{page_content}\n\nSOURCE: {source}\n\n",
+    #    input_variables=["page_content", "source"],
+    #)
+
+    CONTENT_PROMPT = PromptTemplate(
+        template="{page_content}\n\n",#\n\nSOURCE: {source}\n\n",
+        input_variables=["page_content"]
+    )
+
+
+# The main prompt:
+
+    instruction_prompt_template_alpaca_quote = """### Instruction:
+    Quote directly from the SOURCE below that best answers the QUESTION. Only quote full sentences in the correct order. If you cannot find an answer, start your response with "My best guess is: ".
+    
+    CONTENT: {summaries}
+    
+    QUESTION: {question}
+
+    Response:"""
+
+    instruction_prompt_template_orca = """
+    ### System:
+    You are an AI assistant that follows instruction extremely well. Help as much as you can.
+    ### User:
+    Answer the QUESTION using information from the following CONTENT.
+    CONTENT: {summaries}
+    QUESTION: {question}
+
+    ### Response:"""
+
+ 
+
+   
+    INSTRUCTION_PROMPT=PromptTemplate(template=instruction_prompt_template_orca, input_variables=['question', 'summaries'])
+    
+    return INSTRUCTION_PROMPT, CONTENT_PROMPT
+
+def adapt_q_from_chat_history(question, chat_history, extracted_memory, keyword_model=""):#keyword_model): # new_question_keywords, 
+ 
+        chat_history_str, chat_history_first_q, chat_history_first_ans, max_chat_length = _get_chat_history(chat_history)
+
+        if chat_history_str:
+            # Keyword extraction is now done in the add_inputs_to_history function
+            extracted_memory = extracted_memory#remove_q_stopwords(str(chat_history_first_q) + " " + str(chat_history_first_ans))
+            
+           
+            new_question_kworded = str(extracted_memory) + ". " + question #+ " " + new_question_keywords
+            #extracted_memory + " " + question
+            
+        else:
+            new_question_kworded = question #new_question_keywords
+
+        #print("Question output is: " + new_question_kworded)
+            
+        return new_question_kworded
+
+def create_doc_df(docs_keep_out):
+    # Extract content and metadata from 'winning' passages.
+            content=[]
+            meta=[]
+            meta_url=[]
+            page_section=[]
+            score=[]
+
+            for item in docs_keep_out:
+                content.append(item[0].page_content)
+                meta.append(item[0].metadata)
+                meta_url.append(item[0].metadata['source'])
+                page_section.append(item[0].metadata['page_section'])
+                score.append(item[1])       
+
+            # Create df from 'winning' passages
+
+            doc_df = pd.DataFrame(list(zip(content, meta, page_section, meta_url, score)),
+               columns =['page_content', 'metadata', 'page_section', 'meta_url', 'score'])
+
+            docs_content = doc_df['page_content'].astype(str)
+            doc_df['full_url'] = "https://" + doc_df['meta_url'] 
+
+            return doc_df
+
+def hybrid_retrieval(new_question_kworded, k_val, out_passages,
+                           vec_score_cut_off, vec_weight, bm25_weight, svm_weight): # ,vectorstore, embeddings
+
+            vectorstore=globals()["vectorstore"]
+            embeddings=globals()["embeddings"]
+
+
+            docs = vectorstore.similarity_search_with_score(new_question_kworded, k=k_val)
+
+            print("Docs from similarity search:")
+            print(docs)
+
+            # Keep only documents with a certain score
+            docs_len = [len(x[0].page_content) for x in docs]
+            docs_scores = [x[1] for x in docs]
+
+            # Only keep sources that are sufficiently relevant (i.e. similarity search score below threshold below)
+            score_more_limit = pd.Series(docs_scores) < vec_score_cut_off
+            docs_keep = list(compress(docs, score_more_limit))
+
+            if docs_keep == []:
+                docs_keep_as_doc = []
+                docs_content = []
+                docs_url = []
+                return docs_keep_as_doc, docs_content, docs_url
+
+            # Only keep sources that are at least 100 characters long
+            length_more_limit = pd.Series(docs_len) >= 100
+            docs_keep = list(compress(docs_keep, length_more_limit))
+
+            if docs_keep == []:
+                docs_keep_as_doc = []
+                docs_content = []
+                docs_url = []
+                return docs_keep_as_doc, docs_content, docs_url
+
+            docs_keep_as_doc = [x[0] for x in docs_keep]
+            docs_keep_length = len(docs_keep_as_doc)
+
+
+                
+            if docs_keep_length == 1:
+
+                content=[]
+                meta_url=[]
+                score=[]
+                
+                for item in docs_keep:
+                    content.append(item[0].page_content)
+                    meta_url.append(item[0].metadata['source'])
+                    score.append(item[1])       
+
+                # Create df from 'winning' passages
+
+                doc_df = pd.DataFrame(list(zip(content, meta_url, score)),
+                columns =['page_content', 'meta_url', 'score'])
+
+                docs_content = doc_df['page_content'].astype(str)
+                docs_url = doc_df['meta_url']
+
+                return docs_keep_as_doc, docs_content, docs_url
+            
+            # Check for if more docs are removed than the desired output
+            if out_passages > docs_keep_length: 
+                out_passages = docs_keep_length
+                k_val = docs_keep_length
+                     
+            vec_rank = [*range(1, docs_keep_length+1)]
+            vec_score = [(docs_keep_length/x)*vec_weight for x in vec_rank]
+
+            # 2nd level check on retrieved docs with BM25
+
+            content_keep=[]
+            for item in docs_keep:
+                content_keep.append(item[0].page_content)
+
+            corpus = corpus = [doc.lower().split() for doc in content_keep]
+            dictionary = Dictionary(corpus)
+            bm25_model = OkapiBM25Model(dictionary=dictionary)
+            bm25_corpus = bm25_model[list(map(dictionary.doc2bow, corpus))]
+            bm25_index = SparseMatrixSimilarity(bm25_corpus, num_docs=len(corpus), num_terms=len(dictionary),
+                                   normalize_queries=False, normalize_documents=False)
+            query = new_question_kworded.lower().split()
+            tfidf_model = TfidfModel(dictionary=dictionary, smartirs='bnn')  # Enforce binary weighting of queries
+            tfidf_query = tfidf_model[dictionary.doc2bow(query)]
+            similarities = np.array(bm25_index[tfidf_query])
+            #print(similarities)
+            temp = similarities.argsort()
+            ranks = np.arange(len(similarities))[temp.argsort()][::-1]
+
+            # Pair each index with its corresponding value
+            pairs = list(zip(ranks, docs_keep_as_doc))
+            # Sort the pairs by the indices
+            pairs.sort()
+            # Extract the values in the new order
+            bm25_result = [value for ranks, value in pairs]
+            
+            bm25_rank=[]
+            bm25_score = []
+
+            for vec_item in docs_keep:
+                x = 0
+                for bm25_item in bm25_result:
+                    x = x + 1
+                    if bm25_item.page_content == vec_item[0].page_content:
+                        bm25_rank.append(x)
+                        bm25_score.append((docs_keep_length/x)*bm25_weight)
+
+            # 3rd level check on retrieved docs with SVM retriever
+            svm_retriever = SVMRetriever.from_texts(content_keep, embeddings, k = k_val)
+            svm_result = svm_retriever.get_relevant_documents(new_question_kworded)
+
+         
+            svm_rank=[]
+            svm_score = []
+
+            for vec_item in docs_keep:
+                x = 0
+                for svm_item in svm_result:
+                    x = x + 1
+                    if svm_item.page_content == vec_item[0].page_content:
+                        svm_rank.append(x)
+                        svm_score.append((docs_keep_length/x)*svm_weight)
+
+        
+            ## Calculate final score based on three ranking methods
+            final_score = [a  + b + c for a, b, c in zip(vec_score, bm25_score, svm_score)]
+            final_rank = [sorted(final_score, reverse=True).index(x)+1 for x in final_score]
+            # Force final_rank to increment by 1 each time
+            final_rank = list(pd.Series(final_rank).rank(method='first'))
+
+            #print("final rank: " + str(final_rank))
+            #print("out_passages: " + str(out_passages))
+
+            best_rank_index_pos = []
+
+            for x in range(1,out_passages+1):
+                try:
+                    best_rank_index_pos.append(final_rank.index(x))
+                except IndexError: # catch the error
+                    pass
+
+            # Adjust best_rank_index_pos to 
+
+            best_rank_pos_series = pd.Series(best_rank_index_pos)
+
+
+            docs_keep_out = [docs_keep[i] for i in best_rank_index_pos]
+        
+            # Keep only 'best' options
+            docs_keep_as_doc = [x[0] for x in docs_keep_out]
+                               
+            # Make df of best options
+            doc_df = create_doc_df(docs_keep_out)
+
+            return docs_keep_as_doc, doc_df, docs_keep_out
+
+def get_expanded_passages(vectorstore, docs_keep_out, width):
+    """
+    Extracts expanded passages based on given documents and a width for context.
+    
+    Parameters:
+    - vectorstore: The primary data source.
+    - docs_keep_out: List of documents to be expanded.
+    - width: Number of documents to expand around a given document for context.
+    
+    Returns:
+    - expanded_docs: List of expanded Document objects.
+    - doc_df: DataFrame representation of expanded_docs.
+    """
+    
+    def get_docs_from_vstore(vectorstore):
+        vector = vectorstore.docstore._dict
+        return list(vector.items())
+
+    def extract_details(docs_list):
+        docs_list_out = [tup[1] for tup in docs_list]
+        content = [doc.page_content for doc in docs_list_out]
+        meta = [doc.metadata for doc in docs_list_out]
+        return ''.join(content), meta[0], meta[-1]
+    
+    def get_parent_content_and_meta(vstore_docs, width, target):
+        target_range = range(max(0, target - width), min(len(vstore_docs), target + width + 1))
+        parent_vstore_out = [vstore_docs[i] for i in target_range]
+        
+        content_str_out, meta_first_out, meta_last_out = [], [], []
+        for _ in parent_vstore_out:
+            content_str, meta_first, meta_last = extract_details(parent_vstore_out)
+            content_str_out.append(content_str)
+            meta_first_out.append(meta_first)
+            meta_last_out.append(meta_last)
+        return content_str_out, meta_first_out, meta_last_out
+
+    def merge_dicts_except_source(d1, d2):
+            merged = {}
+            for key in d1:
+                if key != "source":
+                    merged[key] = str(d1[key]) + " to " + str(d2[key])
+            else:
+                merged[key] = d1[key]  # or d2[key], based on preference
+            return merged
+
+    def merge_two_lists_of_dicts(list1, list2):
+        return [merge_dicts_except_source(d1, d2) for d1, d2 in zip(list1, list2)]
+
+    vstore_docs = get_docs_from_vstore(vectorstore)
+    parent_vstore_meta_section = [doc.metadata['page_section'] for _, doc in vstore_docs]
+
+    expanded_docs = []
+    for doc, score in docs_keep_out:
+        search_section = doc.metadata['page_section']
+        search_index = parent_vstore_meta_section.index(search_section) if search_section in parent_vstore_meta_section else -1
+        
+        content_str, meta_first, meta_last = get_parent_content_and_meta(vstore_docs, width, search_index)
+        meta_full = merge_two_lists_of_dicts(meta_first, meta_last)
+
+        print(meta_full)
+        
+        expanded_doc = (Document(page_content=content_str[0], metadata=meta_full[0]), score)
+        expanded_docs.append(expanded_doc)
+
+    doc_df = create_doc_df(expanded_docs)  # Assuming you've defined the 'create_doc_df' function elsewhere
+
+    return expanded_docs, doc_df
+
+def create_final_prompt(inputs: Dict[str, str], instruction_prompt, content_prompt, extracted_memory): # , 
+        
+        question =  inputs["question"]
+        chat_history = inputs["chat_history"]
+        
+
+        new_question_kworded = adapt_q_from_chat_history(question, chat_history, extracted_memory) # new_question_keywords, 
+        
+
+        #print("The question passed to the vector search is:")
+        #print(new_question_kworded)
+        
+        #docs_keep_as_doc, docs_content, docs_url = find_relevant_passages(new_question_kworded, k_val = 5, out_passages = 3,
+        #                                                                  vec_score_cut_off = 1.3, vec_weight = 1, tfidf_weight = 0.5, svm_weight = 1)
+
+        docs_keep_as_doc, doc_df, docs_keep_out = hybrid_retrieval(new_question_kworded, k_val = 5, out_passages = 2,
+                                                                          vec_score_cut_off = 1, vec_weight = 1, bm25_weight = 1, svm_weight = 1)#,
+                                                                          #vectorstore=globals()["vectorstore"], embeddings=globals()["embeddings"])
+        
+        # Expand the found passages to the neighbouring context
+        docs_keep_as_doc, doc_df = get_expanded_passages(vectorstore, docs_keep_out, width=1)
+
+        if docs_keep_as_doc == []:
+            {"answer": "I'm sorry, I couldn't find a relevant answer to this question.", "sources":"I'm sorry, I couldn't find a relevant source for this question."}
+        
+        #new_inputs = inputs.copy()
+        #new_inputs["question"] = new_question
+        #new_inputs["chat_history"] = chat_history_str
+        
+        #print(docs_url)
+        #print(doc_df['metadata'])
+
+        # Build up sources content to add to user display
+
+        doc_df['meta_clean'] = [f"<b>{'  '.join(f'{k}: {v}' for k, v in d.items() if k != 'page_section')}</b>" for d in doc_df['metadata']]
+        doc_df['content_meta'] = doc_df['meta_clean'].astype(str) + ".<br><br>" + doc_df['page_content'].astype(str)
+
+        modified_page_content = [f" SOURCE {i+1} - {word}" for i, word in enumerate(doc_df['page_content'])]
+        docs_content_string = ''.join(modified_page_content)
+
+        #docs_content_string = '<br><br>\n\n SOURCE '.join(doc_df['page_content'])#.replace("  "," ")#.strip()
+        sources_docs_content_string = '<br><br>'.join(doc_df['content_meta'])#.replace("  "," ")#.strip()
+        #sources_docs_content_tup = [(sources_docs_content,None)]
+        #print("The draft instruction prompt is:")
+        #print(instruction_prompt)
+        
+        instruction_prompt_out = instruction_prompt.format(question=new_question_kworded, summaries=docs_content_string)
+        #print("The final instruction prompt:")
+        #print(instruction_prompt_out)
+        
+                
+        return instruction_prompt_out, sources_docs_content_string, new_question_kworded
+
+def get_history_sources_final_input_prompt(user_input, history, extracted_memory):#):
+    
+    #if chain_agent is None:
+    #    history.append((user_input, "Please click the button to submit the Huggingface API key before using the chatbot (top right)"))
+    #    return history, history, "", ""
+    print("\n==== date/time: " + str(datetime.datetime.now()) + " ====")
+    print("User input: " + user_input)
+    
+    history = history or []
+    
+
+    
+    # Create instruction prompt
+    instruction_prompt, content_prompt = create_prompt_templates()
+    instruction_prompt_out, docs_content_string, new_question_kworded =\
+                create_final_prompt({"question": user_input, "chat_history": history}, #vectorstore,
+                                    instruction_prompt, content_prompt, extracted_memory)
+    
+  
+    history.append(user_input)
+    
+    print("Output history is:")
+    print(history)
+
+    #print("The output prompt is:")
+    #print(instruction_prompt_out)
+    
+    return history, docs_content_string, instruction_prompt_out
+
+def highlight_found_text_single(search_text:str, full_text:str, hlt_chunk_size:int=hlt_chunk_size, hlt_strat:List=hlt_strat, hlt_overlap:int=hlt_overlap) -> str:
+    """
+    Highlights occurrences of search_text within full_text.
+    
+    Parameters:
+    - search_text (str): The text to be searched for within full_text.
+    - full_text (str): The text within which search_text occurrences will be highlighted.
+    
+    Returns:
+    - str: A string with occurrences of search_text highlighted.
+    
+    Example:
+    >>> highlight_found_text("world", "Hello, world! This is a test. Another world awaits.")
+    'Hello, <mark style="color:black;">world</mark>! This is a test. Another world awaits.'
+    """
+
+    def extract_text_from_input(text,i=0):
+        if isinstance(text, str):
+            return text.replace("  ", " ").strip()#.replace("\r", " ").replace("\n", " ")
+        elif isinstance(text, list):
+            return text[i][0].replace("  ", " ").strip()#.replace("\r", " ").replace("\n", " ")
+        else:
+            return ""
+        
+    def extract_search_text_from_input(text):
+        if isinstance(text, str):
+            return text.replace("  ", " ").strip()#.replace("\r", " ").replace("\n", " ").replace("  ", " ").strip()
+        elif isinstance(text, list):
+            return text[-1][1].replace("  ", " ").strip()#.replace("\r", " ").replace("\n", " ").replace("  ", " ").strip()
+        else:
+            return ""
+            
+    full_text = extract_text_from_input(full_text)
+    search_text = extract_search_text_from_input(search_text)
+
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=hlt_chunk_size,
+        separators=hlt_strat,
+        chunk_overlap=hlt_overlap,
+    )
+    sections = text_splitter.split_text(search_text)
+
+    #print(sections)
+
+    found_positions = {}
+    for x in sections:
+        text_start_pos = full_text.find(x)
+        
+        if text_start_pos != -1:
+            found_positions[text_start_pos] = text_start_pos + len(x)
+
+    # Combine overlapping or adjacent positions
+    sorted_starts = sorted(found_positions.keys())
+    combined_positions = []
+    if sorted_starts:
+        current_start, current_end = sorted_starts[0], found_positions[sorted_starts[0]]
+        for start in sorted_starts[1:]:
+            if start <= (current_end + 1):
+                current_end = max(current_end, found_positions[start])
+            else:
+                combined_positions.append((current_start, current_end))
+                current_start, current_end = start, found_positions[start]
+        combined_positions.append((current_start, current_end))
+
+    # Construct pos_tokens
+    pos_tokens = []
+    prev_end = 0
+    for start, end in combined_positions:
+        pos_tokens.append(full_text[prev_end:start]) # ((full_text[prev_end:start], None))
+        pos_tokens.append('<mark style="color:black;">' + full_text[start:end] + '</mark>')# ("<mark>" + full_text[start:end] + "</mark>",'found')
+        prev_end = end
+    pos_tokens.append(full_text[prev_end:])
+
+    return "".join(pos_tokens)
+
+def highlight_found_text(search_text: str, full_text: str, hlt_chunk_size:int=hlt_chunk_size, hlt_strat:List=hlt_strat, hlt_overlap:int=hlt_overlap) -> str:
+    """
+    Highlights occurrences of search_text within full_text.
+    
+    Parameters:
+    - search_text (str): The text to be searched for within full_text.
+    - full_text (str): The text within which search_text occurrences will be highlighted.
+    
+    Returns:
+    - str: A string with occurrences of search_text highlighted.
+    
+    Example:
+    >>> highlight_found_text("world", "Hello, world! This is a test. Another world awaits.")
+    'Hello, <mark style="color:black;">world</mark>! This is a test. Another <mark style="color:black;">world</mark> awaits.'
+    """
+
+    def extract_text_from_input(text, i=0):
+        if isinstance(text, str):
+            return text.replace("  ", " ").strip()
+        elif isinstance(text, list):
+            return text[i][0].replace("  ", " ").strip()
+        else:
+            return ""
+
+    def extract_search_text_from_input(text):
+        if isinstance(text, str):
+            return text.replace("  ", " ").strip()
+        elif isinstance(text, list):
+            return text[-1][1].replace("  ", " ").strip()
+        else:
+            return ""
+
+    full_text = extract_text_from_input(full_text)
+    search_text = extract_search_text_from_input(search_text)
+
+
+
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=hlt_chunk_size,
+        separators=hlt_strat,
+        chunk_overlap=hlt_overlap,
+    )
+    sections = text_splitter.split_text(search_text)
+
+    found_positions = {}
+    for x in sections:
+        text_start_pos = 0
+        while text_start_pos != -1:
+            text_start_pos = full_text.find(x, text_start_pos)
+            if text_start_pos != -1:
+                found_positions[text_start_pos] = text_start_pos + len(x)
+                text_start_pos += 1
+
+    # Combine overlapping or adjacent positions
+    sorted_starts = sorted(found_positions.keys())
+    combined_positions = []
+    if sorted_starts:
+        current_start, current_end = sorted_starts[0], found_positions[sorted_starts[0]]
+        for start in sorted_starts[1:]:
+            if start <= (current_end + 1):
+                current_end = max(current_end, found_positions[start])
+            else:
+                combined_positions.append((current_start, current_end))
+                current_start, current_end = start, found_positions[start]
+        combined_positions.append((current_start, current_end))
+
+    # Construct pos_tokens
+    pos_tokens = []
+    prev_end = 0
+    for start, end in combined_positions:
+        pos_tokens.append(full_text[prev_end:start])
+        pos_tokens.append('<mark style="color:black;">' + full_text[start:end] + '</mark>')
+        prev_end = end
+    pos_tokens.append(full_text[prev_end:])
+
+    return "".join(pos_tokens)
+
+# # Chat functions
+def produce_streaming_answer_chatbot_gpt4all(history, full_prompt): 
+    
+    print("The question is: ")
+    print(full_prompt)
+
+    # Pull the generated text from the streamer, and update the model output. 
+    history[-1][1] = ""
+    for new_text in gpt4all_model.generate(full_prompt, max_tokens=2000, streaming=True):
+        if new_text == None: new_text = ""
+        history[-1][1] += new_text
+        yield history
+          
+def produce_streaming_answer_chatbot_hf(history, full_prompt): 
+    
+    #print("The question is: ")
+    #print(full_prompt)
+    
+    # Get the model and tokenizer, and tokenize the user text.
+    model_inputs = tokenizer(text=full_prompt, return_tensors="pt").to(torch_device)
+
+    # Start generation on a separate thread, so that we don't block the UI. The text is pulled from the streamer
+    # in the main thread. Adds timeout to the streamer to handle exceptions in the generation thread.
+    streamer = TextIteratorStreamer(tokenizer, timeout=60., skip_prompt=True, skip_special_tokens=True)
+    generate_kwargs = dict(
+        model_inputs,
+        streamer=streamer,
+        max_new_tokens=max_new_tokens,
+        do_sample=sample,
+        repetition_penalty=1.3,
+        top_p=top_p,
+        temperature=temperature,
+        top_k=top_k
+    )
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
+
+    # Pull the generated text from the streamer, and update the model output.
+    import time
+    start = time.time()
+    NUM_TOKENS=0
+    print('-'*4+'Start Generation'+'-'*4)
+
+    history[-1][1] = ""
+    for new_text in streamer:
+        if new_text == None: new_text = ""
+        history[-1][1] += new_text
+        NUM_TOKENS+=1
+        yield history
+        
+    time_generate = time.time() - start
+    print('\n')
+    print('-'*4+'End Generation'+'-'*4)
+    print(f'Num of generated tokens: {NUM_TOKENS}')
+    print(f'Time for complete generation: {time_generate}s')
+    print(f'Tokens per secound: {NUM_TOKENS/time_generate}')
+    print(f'Time per token: {(time_generate/NUM_TOKENS)*1000}ms')
+
+def produce_streaming_answer_chatbot_ctrans(history, full_prompt): 
+    
+    print("The question is: ")
+    print(full_prompt)
+
+    #tokens = ctrans_llm.tokenize(full_prompt)
+
+    #import psutil
+    #from loguru import logger
+
+    #_ = [elm for elm in full_prompt.splitlines() if elm.strip()]
+    #stop_string = [elm.split(":")[0] + ":" for elm in _][-2]
+    #print(stop_string)
+
+    #logger.debug(f"{stop_string=} not used")
+
+    #_ = psutil.cpu_count(logical=False) - 1
+    #cpu_count: int = int(_) if _ else 1
+    #logger.debug(f"{cpu_count=}")
+
+    # Pull the generated text from the streamer, and update the model output.
+    config = GenerationConfig(reset=True)
+    history[-1][1] = ""
+    for new_text in ctrans_generate(prompt=full_prompt, config=config):
+        if new_text == None: new_text = ""
+        history[-1][1] += new_text
+        yield history
+
+@dataclass
+class GenerationConfig:
+    temperature: float = temperature
+    top_k: int = top_k
+    top_p: float = top_p
+    repetition_penalty: float = repetition_penalty
+    last_n_tokens: int = last_n_tokens
+    max_new_tokens: int = max_new_tokens
+    #seed: int = 42
+    reset: bool = reset
+    stream: bool = stream
+    threads: int = threads
+    batch_size:int = batch_size
+    #context_length:int = context_length
+    #gpu_layers:int = gpu_layers
+    #stop: list[str] = field(default_factory=lambda: [stop_string])
+
+def ctrans_generate(
+    prompt: str,
+    llm=ctrans_llm,
+    config: GenerationConfig = GenerationConfig(),
+):
+    """Run model inference, will return a Generator if streaming is true."""
+
+    return llm(
+        prompt,
+        **asdict(config),
+    )
+
+def turn_off_interactivity(user_message, history):
+        return gr.update(value="", interactive=False), history + [[user_message, None]]
+
+# # Chat history functions
+
+def clear_chat(chat_history_state, sources, chat_message, current_topic):
+    chat_history_state = []
+    sources = ''
+    chat_message = ''
+    current_topic = ''
+
+    return chat_history_state, sources, chat_message, current_topic
+
+def _get_chat_history(chat_history: List[Tuple[str, str]], max_chat_length:int = 20): # Limit to last x interactions only
+
+    if not chat_history:
+        chat_history = []
+
+    if len(chat_history) > max_chat_length:
+        chat_history = chat_history[-max_chat_length:]
+        
+    #print(chat_history)
+
+    first_q = ""
+    first_ans = ""
+    for human_s, ai_s in chat_history:
+        first_q = human_s
+        first_ans = ai_s
+
+        #print("Text to keyword extract: " + first_q + " " + first_ans)
+        break
+
+    conversation = ""
+    for human_s, ai_s in chat_history:
+        human = f"Human: " + human_s
+        ai = f"Assistant: " + ai_s
+        conversation += "\n" + "\n".join([human, ai])
+
+    return conversation, first_q, first_ans, max_chat_length
+
+def add_inputs_answer_to_history(user_message, history, current_topic):
+    
+    #history.append((user_message, [-1]))
+
+    chat_history_str, chat_history_first_q, chat_history_first_ans, max_chat_length = _get_chat_history(history)
+
+
+    # Only get the keywords for the first question and response, or do it every time if over 'max_chat_length' responses in the conversation
+    if (len(history) == 1) | (len(history) > max_chat_length):
+        
+        #print("History after appending is:")
+        #print(history)
+
+        first_q_and_first_ans = str(chat_history_first_q) + " " + str(chat_history_first_ans)
+        #ner_memory = remove_q_ner_extractor(first_q_and_first_ans)
+        keywords = keybert_keywords(first_q_and_first_ans, n = 8, kw_model=kw_model)
+        #keywords.append(ner_memory)
+
+        # Remove duplicate words while preserving order
+        ordered_tokens = set()
+        result = []
+        for word in keywords:
+                if word not in ordered_tokens:
+                        ordered_tokens.add(word)
+                        result.append(word)
+
+        extracted_memory = ' '.join(result)
+
+    else: extracted_memory=current_topic
+    
+    print("Extracted memory is:")
+    print(extracted_memory)
+    
+    
+    return history, extracted_memory
+
+def remove_q_stopwords(question): # Remove stopwords from question. Not used at the moment 
+    # Prepare keywords from question by removing stopwords
+    text = question.lower()
+
+    # Remove numbers
+    text = re.sub('[0-9]', '', text)
+
+    tokenizer = RegexpTokenizer(r'\w+')
+    text_tokens = tokenizer.tokenize(text)
+    #text_tokens = word_tokenize(text)
+    tokens_without_sw = [word for word in text_tokens if not word in stopwords]
+
+    # Remove duplicate words while preserving order
+    ordered_tokens = set()
+    result = []
+    for word in tokens_without_sw:
+        if word not in ordered_tokens:
+            ordered_tokens.add(word)
+            result.append(word)
+     
+
+
+    new_question_keywords = ' '.join(result)
+    return new_question_keywords
+
+def remove_q_ner_extractor(question):
+    
+    predict_out = ner_model.predict(question)
+
+
+
+    predict_tokens = [' '.join(v for k, v in d.items() if k == 'span') for d in predict_out]
+
+    # Remove duplicate words while preserving order
+    ordered_tokens = set()
+    result = []
+    for word in predict_tokens:
+        if word not in ordered_tokens:
+            ordered_tokens.add(word)
+            result.append(word)
+     
+
+
+    new_question_keywords = ' '.join(result).lower()
+    return new_question_keywords
+
+def apply_lemmatize(text, wnl=WordNetLemmatizer()):
+
+    def prep_for_lemma(text):
+
+        # Remove numbers
+        text = re.sub('[0-9]', '', text)
+        print(text)
+
+        tokenizer = RegexpTokenizer(r'\w+')
+        text_tokens = tokenizer.tokenize(text)
+        #text_tokens = word_tokenize(text)
+
+        return text_tokens
+
+    tokens = prep_for_lemma(text)
+
+    def lem_word(word):
+    
+        if len(word) > 3: out_word = wnl.lemmatize(word)
+        else: out_word = word
+
+        return out_word
+
+    return [lem_word(token) for token in tokens]
+
+def keybert_keywords(text, n, kw_model):
+    tokens_lemma = apply_lemmatize(text)
+    lemmatised_text = ' '.join(tokens_lemma)
+
+    keywords_text = keybert.KeyBERT(model=kw_model).extract_keywords(lemmatised_text, stop_words='english', top_n=n, 
+                                                   keyphrase_ngram_range=(1, 1))
+    keywords_list = [item[0] for item in keywords_text]
+
+    return keywords_list
+

chatfuncs/ingest.py

@@ -0,0 +1,522 @@
+# ---
+# jupyter:
+#   jupytext:
+#     formats: ipynb,py:light
+#     text_representation:
+#       extension: .py
+#       format_name: light
+#       format_version: '1.5'
+#       jupytext_version: 1.14.6
+#   kernelspec:
+#     display_name: Python 3 (ipykernel)
+#     language: python
+#     name: python3
+# ---
+
+# # Ingest website to FAISS
+
+# ## Install/ import stuff we need
+
+import os
+from pathlib import Path
+import re
+import requests
+import pandas as pd
+import dateutil.parser
+from typing import TypeVar, List
+
+from langchain.embeddings import HuggingFaceInstructEmbeddings, HuggingFaceEmbeddings
+from langchain.vectorstores.faiss import FAISS
+from langchain.vectorstores import Chroma
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.docstore.document import Document
+
+from bs4 import BeautifulSoup
+from docx import Document as Doc
+from pypdf import PdfReader
+
+PandasDataFrame = TypeVar('pd.core.frame.DataFrame')
+# -
+
+split_strat = ["\n\n", "\n", ".", "!", "?",  ","]
+chunk_size = 500
+chunk_overlap = 0
+start_index = True
+
+## Parse files
+
+def parse_file(file_paths, div:str = "p"):
+    """
+    Accepts a list of file paths, determines each file's type based on its extension,
+    and passes it to the relevant parsing function.
+    
+    Parameters:
+        file_paths (list): List of file paths.
+        div (str): (optional) Div to pull out of html file/url with BeautifulSoup
+    
+    Returns:
+        dict: A dictionary with file paths as keys and their parsed content (or error message) as values.
+    """
+    
+    def determine_file_type(file_path):
+        """
+        Determine the file type based on its extension.
+    
+        Parameters:
+            file_path (str): Path to the file.
+    
+        Returns:
+            str: File extension (e.g., '.pdf', '.docx', '.txt', '.html').
+        """
+        return os.path.splitext(file_path)[1].lower()
+
+    if not isinstance(file_paths, list):
+        raise ValueError("Expected a list of file paths.")
+    
+    extension_to_parser = {
+        '.pdf': parse_pdf,
+        '.docx': parse_docx,
+        '.txt': parse_txt,
+        '.html': parse_html,
+        '.htm': parse_html  # Considering both .html and .htm for HTML files
+    }
+    
+    parsed_contents = {}
+
+    for file_path in file_paths:
+        print(file_path.name)
+        #file = open(file_path.name, 'r')
+        #print(file)
+        file_extension = determine_file_type(file_path.name)
+        if file_extension in extension_to_parser:
+            parsed_contents[file_path.name] = extension_to_parser[file_extension](file_path.name)
+        else:
+            parsed_contents[file_path.name] = f"Unsupported file type: {file_extension}"
+    
+    return parsed_contents
+
+def text_regex_clean(text):
+    # Merge hyphenated words
+        text = re.sub(r"(\w+)-\n(\w+)", r"\1\2", text)
+        # If a double newline ends in a letter, add a full stop.
+        text = re.sub(r'(?<=[a-zA-Z])\n\n', '.\n\n', text)
+        # Fix newlines in the middle of sentences
+        text = re.sub(r"(?<!\n\s)\n(?!\s\n)", " ", text.strip())
+        # Remove multiple newlines
+        text = re.sub(r"\n\s*\n", "\n\n", text)
+        text = re.sub(r"  ", " ", text)
+        # Add full stops and new lines between words with no space between where the second one has a capital letter
+        text = re.sub(r'(?<=[a-z])(?=[A-Z])', '. \n\n', text)
+
+        return text
+
+def parse_pdf(file) -> List[str]:
+
+    """
+    Extract text from a PDF file.
+    
+    Parameters:
+        file_path (str): Path to the PDF file.
+    
+    Returns:
+        List[str]: Extracted text from the PDF.
+    """
+    
+    output = []
+    #for file in files:
+    print(file) # .name
+    pdf = PdfReader(file) #[i] .name[i]
+    
+    for page in pdf.pages:
+            text = page.extract_text()
+            
+            text = text_regex_clean(text)
+
+            output.append(text)
+    return output
+
+def parse_docx(file_path):
+    """
+    Reads the content of a .docx file and returns it as a string.
+
+    Parameters:
+    - file_path (str): Path to the .docx file.
+
+    Returns:
+    - str: Content of the .docx file.
+    """
+    doc = Doc(file_path)
+    full_text = []
+    for para in doc.paragraphs:
+        para = text_regex_clean(para)
+
+        full_text.append(para.text.replace("  ", " ").strip())
+    return '\n'.join(full_text)
+
+def parse_txt(file_path):
+    """
+    Read text from a TXT or HTML file.
+    
+    Parameters:
+        file_path (str): Path to the TXT or HTML file.
+    
+    Returns:
+        str: Text content of the file.
+    """
+    with open(file_path, 'r', encoding="utf-8") as file:
+        file_contents = file.read().replace("  ", " ").strip()
+
+        file_contents = text_regex_clean(file_contents)
+
+        return file_contents
+
+def parse_html(page_url, div_filter="p"):
+    """
+    Determine if the source is a web URL or a local HTML file, extract the content based on the div of choice. Also tries to extract dates (WIP)
+
+    Parameters:
+        page_url (str): The web URL or local file path.
+
+    Returns:
+        str: Extracted content.
+    """
+
+    def is_web_url(s):
+        """
+        Check if the input string is a web URL.
+        """
+        return s.startswith("http://") or s.startswith("https://")
+
+    def is_local_html_file(s):
+        """
+        Check if the input string is a path to a local HTML file.
+        """
+        return (s.endswith(".html") or s.endswith(".htm")) and os.path.isfile(s)
+
+    def extract_text_from_source(source):
+        """
+        Determine if the source is a web URL or a local HTML file, 
+        and then extract its content accordingly.
+
+        Parameters:
+        source (str): The web URL or local file path.
+
+        Returns:
+        str: Extracted content.
+        """
+        if is_web_url(source):
+            response = requests.get(source)
+            response.raise_for_status()  # Raise an HTTPError for bad responses
+            return response.text.replace("  ", " ").strip()
+        elif is_local_html_file(source):
+            with open(source, 'r', encoding='utf-8') as file:
+                file_out = file.read().replace
+                return file_out
+        else:
+            raise ValueError("Input is neither a valid web URL nor a local HTML file path.")
+               
+
+    def clean_html_data(data, date_filter="", div_filt="p"):
+        """
+        Extracts and cleans data from HTML content.
+
+        Parameters:
+            data (str): HTML content to be parsed.
+            date_filter (str, optional): Date string to filter results. If set, only content with a date greater than this will be returned.
+            div_filt (str, optional): HTML tag to search for text content. Defaults to "p".
+
+        Returns:
+            tuple: Contains extracted text and date as strings. Returns empty strings if not found.
+        """
+    
+        soup = BeautifulSoup(data, 'html.parser')
+
+        # Function to exclude div with id "bar"
+        def exclude_div_with_id_bar(tag):
+            return tag.has_attr('id') and tag['id'] == 'related-links'
+
+        text_elements = soup.find_all(div_filt)
+        date_elements = soup.find_all(div_filt, {"class": "page-neutral-intro__meta"})
+    
+        # Extract date
+        date_out = ""
+        if date_elements:
+            date_out = re.search(">(.*?)<", str(date_elements[0])).group(1)
+            date_dt = dateutil.parser.parse(date_out)
+
+            if date_filter:
+                date_filter_dt = dateutil.parser.parse(date_filter)
+                if date_dt < date_filter_dt:
+                    return '', date_out
+
+        # Extract text
+        text_out_final = ""
+        if text_elements:
+            text_out_final = '\n'.join(paragraph.text for paragraph in text_elements)
+            text_out_final = text_regex_clean(text_out_final)
+        else:
+            print(f"No elements found with tag '{div_filt}'. No text returned.")
+    
+        return text_out_final, date_out
+    
+
+    #page_url = "https://pypi.org/project/InstructorEmbedding/" #'https://www.ons.gov.uk/visualisations/censusareachanges/E09000022/index.html'
+
+    html_text = extract_text_from_source(page_url)
+    #print(page.text)
+
+    texts = []
+    metadatas = []
+
+    clean_text, date = clean_html_data(html_text, date_filter="", div_filt=div_filter)
+    texts.append(clean_text)
+    metadatas.append({"source": page_url, "date":str(date)})
+
+    return texts, metadatas
+
+# +
+# Convert parsed text to docs
+# -
+
+def text_to_docs(text_dict: dict, chunk_size: int = chunk_size) -> List[Document]:
+    """
+    Converts the output of parse_file (a dictionary of file paths to content)
+    to a list of Documents with metadata.
+    """
+    
+    doc_sections = []
+    parent_doc_sections = []
+
+    for file_path, content in text_dict.items():
+        ext = os.path.splitext(file_path)[1].lower()
+
+        # Depending on the file extension, handle the content
+        if ext == '.pdf':
+            docs, page_docs = pdf_text_to_docs(content, chunk_size)
+        elif ext in ['.html', '.htm', '.txt', '.docx']:
+            # Assuming you want to process HTML similarly to PDF in this context
+            docs = html_text_to_docs(content, chunk_size)
+        else:
+            print(f"Unsupported file type {ext} for {file_path}. Skipping.")
+            continue
+
+
+        match = re.search(r'.*[\/\\](.+)$', file_path)
+        filename_end = match.group(1)
+
+        # Add filename as metadata
+        for doc in docs: doc.metadata["source"] = filename_end
+        #for parent_doc in parent_docs: parent_doc.metadata["source"] = filename_end
+        
+        doc_sections.extend(docs)
+        #parent_doc_sections.extend(parent_docs)
+
+    return doc_sections, page_docs
+
+def pdf_text_to_docs(text, chunk_size: int = chunk_size) -> List[Document]:
+    """Converts a string or list of strings to a list of Documents
+    with metadata."""
+
+    #print(text)
+
+    if isinstance(text, str):
+        # Take a single string as one page
+        text = [text]
+        
+    page_docs = [Document(page_content=page, metadata={"page": page}) for page in text]
+  
+
+    # Add page numbers as metadata
+    for i, doc in enumerate(page_docs):
+        doc.metadata["page"] = i + 1
+
+    print("page docs are: ")
+    print(page_docs)
+
+    # Split pages into sections
+    doc_sections = []
+
+    for doc in page_docs:
+
+        #print("page content: ")
+        #print(doc.page_content)
+
+        if doc.page_content == '':
+            sections = ['']
+
+        else:
+            text_splitter = RecursiveCharacterTextSplitter(
+                chunk_size=chunk_size,
+                separators=split_strat,#["\n\n", "\n", ".", "!", "?", ",", " ", ""],
+                chunk_overlap=chunk_overlap,
+                add_start_index=True
+            )
+            sections = text_splitter.split_text(doc.page_content)
+        
+        for i, section in enumerate(sections):
+            doc = Document(
+                   page_content=section, metadata={"page": doc.metadata["page"], "section": i, "page_section": f"{doc.metadata['page']}-{i}"})
+
+            
+            doc_sections.append(doc)
+
+    return doc_sections, page_docs#, parent_doc
+
+def html_text_to_docs(texts, metadatas, chunk_size:int = chunk_size):
+
+    text_splitter = RecursiveCharacterTextSplitter(
+        separators=split_strat,#["\n\n", "\n", ".", "!", "?", ",", " ", ""],
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        length_function=len,
+        add_start_index=True
+    )
+
+    #print(texts)
+    #print(metadatas)
+
+    documents = text_splitter.create_documents(texts, metadatas=metadatas)
+
+    for i, section in enumerate(documents):
+        section.metadata["section"] = i + 1
+
+    return documents
+
+# # Functions for working with documents after loading them back in
+
+def pull_out_data(series):
+
+    # define a lambda function to convert each string into a tuple
+    to_tuple = lambda x: eval(x)
+
+    # apply the lambda function to each element of the series
+    series_tup = series.apply(to_tuple)
+
+    series_tup_content = list(zip(*series_tup))[1]
+
+    series = pd.Series(list(series_tup_content))#.str.replace("^Main post content", "", regex=True).str.strip()
+
+    return series
+
+def docs_from_csv(df):
+
+    import ast
+    
+    documents = []
+    
+    page_content = pull_out_data(df["0"])
+    metadatas = pull_out_data(df["1"])
+
+    for x in range(0,len(df)):       
+        new_doc = Document(page_content=page_content[x], metadata=metadatas[x])
+        documents.append(new_doc)
+        
+    return documents
+
+def docs_from_lists(docs, metadatas):
+
+    documents = []
+
+    for x, doc in enumerate(docs):
+        new_doc = Document(page_content=doc, metadata=metadatas[x])
+        documents.append(new_doc)
+        
+    return documents
+
+def docs_elements_from_csv_save(docs_path="documents.csv"):
+
+    documents = pd.read_csv(docs_path)
+
+    docs_out = docs_from_csv(documents)
+
+    out_df = pd.DataFrame(docs_out)
+
+    docs_content = pull_out_data(out_df[0].astype(str))
+
+    docs_meta = pull_out_data(out_df[1].astype(str))
+
+    doc_sources = [d['source'] for d in docs_meta]
+
+    return out_df, docs_content, docs_meta, doc_sources
+
+# ## Create embeddings and save faiss vector store to the path specified in `save_to`
+
+def load_embeddings(model_name = "thenlper/gte-base"):
+
+    if model_name == "hkunlp/instructor-large":
+        embeddings_func = HuggingFaceInstructEmbeddings(model_name=model_name,
+        embed_instruction="Represent the paragraph for retrieval: ",
+        query_instruction="Represent the question for retrieving supporting documents: "
+        )
+
+    else: 
+        embeddings_func = HuggingFaceEmbeddings(model_name=model_name)
+
+    global embeddings
+
+    embeddings = embeddings_func
+
+    #return embeddings_func
+
+def embed_faiss_save_to_zip(docs_out, save_to="faiss_lambeth_census_embedding", model_name = "thenlper/gte-base"):
+
+    load_embeddings(model_name=model_name)
+
+    #embeddings_fast = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+
+    print(f"> Total split documents: {len(docs_out)}")
+
+    vectorstore = FAISS.from_documents(documents=docs_out, embedding=embeddings)
+        
+
+    if Path(save_to).exists():
+        vectorstore.save_local(folder_path=save_to)
+
+    print("> DONE")
+    print(f"> Saved to: {save_to}")
+
+    ### Save as zip, then remove faiss/pkl files to allow for upload to huggingface
+
+    import shutil
+
+    shutil.make_archive(save_to, 'zip', save_to)
+
+    os.remove(save_to + "/index.faiss")
+    os.remove(save_to + "/index.pkl")
+
+    shutil.move(save_to + '.zip', save_to + "/" + save_to + '.zip')
+
+    return vectorstore
+
+def docs_to_chroma_save(embeddings, docs_out:PandasDataFrame, save_to:str):
+    print(f"> Total split documents: {len(docs_out)}")
+    
+    vectordb = Chroma.from_documents(documents=docs_out, 
+                                 embedding=embeddings,
+                                 persist_directory=save_to)
+    
+    # persiste the db to disk
+    vectordb.persist()
+    
+    print("> DONE")
+    print(f"> Saved to: {save_to}")
+    
+    return vectordb
+
+def sim_search_local_saved_vec(query, k_val, save_to="faiss_lambeth_census_embedding"):
+
+    load_embeddings()
+
+    docsearch = FAISS.load_local(folder_path=save_to, embeddings=embeddings)
+
+
+    display(Markdown(question))
+
+    search = docsearch.similarity_search_with_score(query, k=k_val)
+
+    for item in search:
+        print(item[0].page_content)
+        print(f"Page: {item[0].metadata['source']}")
+        print(f"Date: {item[0].metadata['date']}")
+        print(f"Score: {item[1]}")
+        print("---")

chatfuncs/ingest_borough_plan.py

@@ -0,0 +1,16 @@
+import ingest as ing
+import pandas as pd
+
+borough_plan_text = ing.parse_file([open("faiss_embedding/Lambeth_2030-Our_Future_Our_Lambeth.pdf")])
+print("Borough plan text created")
+
+#print(borough_plan_text)
+
+borough_plan_docs, borough_plan_page_docs = ing.text_to_docs(borough_plan_text)
+print("Borough plan docs created")
+
+embedding_model = "thenlper/gte-base"
+
+ing.load_embeddings(model_name = embedding_model)
+ing.embed_faiss_save_to_zip(borough_plan_docs, save_to="faiss_embedding", model_name = embedding_model)
+#borough_plan_parent_docs.to_csv("borough_plan_parent_docs.csv", index=False)

requirements.txt

@@ -0,0 +1,17 @@
+langchain
+beautifulsoup4
+pandas
+black
+isort
+Flask
+transformers
+--extra-index-url https://download.pytorch.org/whl/cu113
+torch
+sentence_transformers
+faiss-cpu
+bitsandbytes
+accelerate
+optimum
+pypdf
+gradio
+gradio_client==0.2.7

test/sample.html

@@ -0,0 +1,769 @@
+<html xmlns:v="urn:schemas-microsoft-com:vml"
+xmlns:o="urn:schemas-microsoft-com:office:office"
+xmlns:w="urn:schemas-microsoft-com:office:word"
+xmlns:m="http://schemas.microsoft.com/office/2004/12/omml"
+xmlns="http://www.w3.org/TR/REC-html40">
+
+<head>
+<meta http-equiv=Content-Type content="text/html; charset=windows-1252">
+<meta name=ProgId content=Word.Document>
+<meta name=Generator content="Microsoft Word 15">
+<meta name=Originator content="Microsoft Word 15">
+<link rel=File-List href="sample_files/filelist.xml">
+<!--[if gte mso 9]><xml>
+ <o:DocumentProperties>
+  <o:Author>Sean Pedrick-Case</o:Author>
+  <o:LastAuthor>Sean Pedrick-Case</o:LastAuthor>
+  <o:Revision>2</o:Revision>
+  <o:TotalTime>1</o:TotalTime>
+  <o:Created>2023-08-07T09:40:00Z</o:Created>
+  <o:LastSaved>2023-08-07T09:40:00Z</o:LastSaved>
+  <o:Pages>1</o:Pages>
+  <o:Words>2</o:Words>
+  <o:Characters>12</o:Characters>
+  <o:Lines>1</o:Lines>
+  <o:Paragraphs>1</o:Paragraphs>
+  <o:CharactersWithSpaces>13</o:CharactersWithSpaces>
+  <o:Version>16.00</o:Version>
+ </o:DocumentProperties>
+ <o:OfficeDocumentSettings>
+  <o:AllowPNG/>
+ </o:OfficeDocumentSettings>
+</xml><![endif]-->
+<link rel=themeData href="sample_files/themedata.thmx">
+<link rel=colorSchemeMapping href="sample_files/colorschememapping.xml">
+<!--[if gte mso 9]><xml>
+ <w:WordDocument>
+  <w:SpellingState>Clean</w:SpellingState>
+  <w:GrammarState>Clean</w:GrammarState>
+  <w:TrackMoves/>
+  <w:TrackFormatting/>
+  <w:PunctuationKerning/>
+  <w:ValidateAgainstSchemas/>
+  <w:SaveIfXMLInvalid>false</w:SaveIfXMLInvalid>
+  <w:IgnoreMixedContent>false</w:IgnoreMixedContent>
+  <w:AlwaysShowPlaceholderText>false</w:AlwaysShowPlaceholderText>
+  <w:DoNotPromoteQF/>
+  <w:LidThemeOther>EN-GB</w:LidThemeOther>
+  <w:LidThemeAsian>X-NONE</w:LidThemeAsian>
+  <w:LidThemeComplexScript>X-NONE</w:LidThemeComplexScript>
+  <w:Compatibility>
+   <w:BreakWrappedTables/>
+   <w:SnapToGridInCell/>
+   <w:WrapTextWithPunct/>
+   <w:UseAsianBreakRules/>
+   <w:DontGrowAutofit/>
+   <w:SplitPgBreakAndParaMark/>
+   <w:EnableOpenTypeKerning/>
+   <w:DontFlipMirrorIndents/>
+   <w:OverrideTableStyleHps/>
+  </w:Compatibility>
+  <m:mathPr>
+   <m:mathFont m:val="Cambria Math"/>
+   <m:brkBin m:val="before"/>
+   <m:brkBinSub m:val="&#45;-"/>
+   <m:smallFrac m:val="off"/>
+   <m:dispDef/>
+   <m:lMargin m:val="0"/>
+   <m:rMargin m:val="0"/>
+   <m:defJc m:val="centerGroup"/>
+   <m:wrapIndent m:val="1440"/>
+   <m:intLim m:val="subSup"/>
+   <m:naryLim m:val="undOvr"/>
+  </m:mathPr></w:WordDocument>
+</xml><![endif]--><!--[if gte mso 9]><xml>
+ <w:LatentStyles DefLockedState="false" DefUnhideWhenUsed="false"
+  DefSemiHidden="false" DefQFormat="false" DefPriority="99"
+  LatentStyleCount="376">
+  <w:LsdException Locked="false" Priority="0" QFormat="true" Name="Normal"/>
+  <w:LsdException Locked="false" Priority="9" QFormat="true" Name="heading 1"/>
+  <w:LsdException Locked="false" Priority="9" SemiHidden="true"
+   UnhideWhenUsed="true" QFormat="true" Name="heading 2"/>
+  <w:LsdException Locked="false" Priority="9" SemiHidden="true"
+   UnhideWhenUsed="true" QFormat="true" Name="heading 3"/>
+  <w:LsdException Locked="false" Priority="9" SemiHidden="true"
+   UnhideWhenUsed="true" QFormat="true" Name="heading 4"/>
+  <w:LsdException Locked="false" Priority="9" SemiHidden="true"
+   UnhideWhenUsed="true" QFormat="true" Name="heading 5"/>
+  <w:LsdException Locked="false" Priority="9" SemiHidden="true"
+   UnhideWhenUsed="true" QFormat="true" Name="heading 6"/>
+  <w:LsdException Locked="false" Priority="9" SemiHidden="true"
+   UnhideWhenUsed="true" QFormat="true" Name="heading 7"/>
+  <w:LsdException Locked="false" Priority="9" SemiHidden="true"
+   UnhideWhenUsed="true" QFormat="true" Name="heading 8"/>
+  <w:LsdException Locked="false" Priority="9" SemiHidden="true"
+   UnhideWhenUsed="true" QFormat="true" Name="heading 9"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="index 1"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="index 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="index 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="index 4"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="index 5"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="index 6"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="index 7"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="index 8"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="index 9"/>
+  <w:LsdException Locked="false" Priority="39" SemiHidden="true"
+   UnhideWhenUsed="true" Name="toc 1"/>
+  <w:LsdException Locked="false" Priority="39" SemiHidden="true"
+   UnhideWhenUsed="true" Name="toc 2"/>
+  <w:LsdException Locked="false" Priority="39" SemiHidden="true"
+   UnhideWhenUsed="true" Name="toc 3"/>
+  <w:LsdException Locked="false" Priority="39" SemiHidden="true"
+   UnhideWhenUsed="true" Name="toc 4"/>
+  <w:LsdException Locked="false" Priority="39" SemiHidden="true"
+   UnhideWhenUsed="true" Name="toc 5"/>
+  <w:LsdException Locked="false" Priority="39" SemiHidden="true"
+   UnhideWhenUsed="true" Name="toc 6"/>
+  <w:LsdException Locked="false" Priority="39" SemiHidden="true"
+   UnhideWhenUsed="true" Name="toc 7"/>
+  <w:LsdException Locked="false" Priority="39" SemiHidden="true"
+   UnhideWhenUsed="true" Name="toc 8"/>
+  <w:LsdException Locked="false" Priority="39" SemiHidden="true"
+   UnhideWhenUsed="true" Name="toc 9"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Normal Indent"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="footnote text"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="annotation text"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="header"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="footer"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="index heading"/>
+  <w:LsdException Locked="false" Priority="35" SemiHidden="true"
+   UnhideWhenUsed="true" QFormat="true" Name="caption"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="table of figures"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="envelope address"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="envelope return"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="footnote reference"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="annotation reference"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="line number"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="page number"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="endnote reference"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="endnote text"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="table of authorities"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="macro"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="toa heading"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Bullet"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Number"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List 4"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List 5"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Bullet 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Bullet 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Bullet 4"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Bullet 5"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Number 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Number 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Number 4"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Number 5"/>
+  <w:LsdException Locked="false" Priority="10" QFormat="true" Name="Title"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Closing"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Signature"/>
+  <w:LsdException Locked="false" Priority="1" SemiHidden="true"
+   UnhideWhenUsed="true" Name="Default Paragraph Font"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Body Text"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Body Text Indent"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Continue"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Continue 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Continue 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Continue 4"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="List Continue 5"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Message Header"/>
+  <w:LsdException Locked="false" Priority="11" QFormat="true" Name="Subtitle"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Salutation"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Date"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Body Text First Indent"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Body Text First Indent 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Note Heading"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Body Text 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Body Text 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Body Text Indent 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Body Text Indent 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Block Text"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Hyperlink"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="FollowedHyperlink"/>
+  <w:LsdException Locked="false" Priority="22" QFormat="true" Name="Strong"/>
+  <w:LsdException Locked="false" Priority="20" QFormat="true" Name="Emphasis"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Document Map"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Plain Text"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="E-mail Signature"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="HTML Top of Form"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="HTML Bottom of Form"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Normal (Web)"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="HTML Acronym"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="HTML Address"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="HTML Cite"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="HTML Code"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="HTML Definition"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="HTML Keyboard"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="HTML Preformatted"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="HTML Sample"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="HTML Typewriter"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="HTML Variable"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Normal Table"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="annotation subject"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="No List"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Outline List 1"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Outline List 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Outline List 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Simple 1"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Simple 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Simple 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Classic 1"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Classic 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Classic 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Classic 4"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Colorful 1"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Colorful 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Colorful 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Columns 1"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Columns 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Columns 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Columns 4"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Columns 5"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Grid 1"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Grid 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Grid 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Grid 4"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Grid 5"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Grid 6"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Grid 7"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Grid 8"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table List 1"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table List 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table List 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table List 4"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table List 5"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table List 6"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table List 7"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table List 8"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table 3D effects 1"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table 3D effects 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table 3D effects 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Contemporary"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Elegant"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Professional"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Subtle 1"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Subtle 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Web 1"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Web 2"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Web 3"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Balloon Text"/>
+  <w:LsdException Locked="false" Priority="39" Name="Table Grid"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Table Theme"/>
+  <w:LsdException Locked="false" SemiHidden="true" Name="Placeholder Text"/>
+  <w:LsdException Locked="false" Priority="1" QFormat="true" Name="No Spacing"/>
+  <w:LsdException Locked="false" Priority="60" Name="Light Shading"/>
+  <w:LsdException Locked="false" Priority="61" Name="Light List"/>
+  <w:LsdException Locked="false" Priority="62" Name="Light Grid"/>
+  <w:LsdException Locked="false" Priority="63" Name="Medium Shading 1"/>
+  <w:LsdException Locked="false" Priority="64" Name="Medium Shading 2"/>
+  <w:LsdException Locked="false" Priority="65" Name="Medium List 1"/>
+  <w:LsdException Locked="false" Priority="66" Name="Medium List 2"/>
+  <w:LsdException Locked="false" Priority="67" Name="Medium Grid 1"/>
+  <w:LsdException Locked="false" Priority="68" Name="Medium Grid 2"/>
+  <w:LsdException Locked="false" Priority="69" Name="Medium Grid 3"/>
+  <w:LsdException Locked="false" Priority="70" Name="Dark List"/>
+  <w:LsdException Locked="false" Priority="71" Name="Colorful Shading"/>
+  <w:LsdException Locked="false" Priority="72" Name="Colorful List"/>
+  <w:LsdException Locked="false" Priority="73" Name="Colorful Grid"/>
+  <w:LsdException Locked="false" Priority="60" Name="Light Shading Accent 1"/>
+  <w:LsdException Locked="false" Priority="61" Name="Light List Accent 1"/>
+  <w:LsdException Locked="false" Priority="62" Name="Light Grid Accent 1"/>
+  <w:LsdException Locked="false" Priority="63" Name="Medium Shading 1 Accent 1"/>
+  <w:LsdException Locked="false" Priority="64" Name="Medium Shading 2 Accent 1"/>
+  <w:LsdException Locked="false" Priority="65" Name="Medium List 1 Accent 1"/>
+  <w:LsdException Locked="false" SemiHidden="true" Name="Revision"/>
+  <w:LsdException Locked="false" Priority="34" QFormat="true"
+   Name="List Paragraph"/>
+  <w:LsdException Locked="false" Priority="29" QFormat="true" Name="Quote"/>
+  <w:LsdException Locked="false" Priority="30" QFormat="true"
+   Name="Intense Quote"/>
+  <w:LsdException Locked="false" Priority="66" Name="Medium List 2 Accent 1"/>
+  <w:LsdException Locked="false" Priority="67" Name="Medium Grid 1 Accent 1"/>
+  <w:LsdException Locked="false" Priority="68" Name="Medium Grid 2 Accent 1"/>
+  <w:LsdException Locked="false" Priority="69" Name="Medium Grid 3 Accent 1"/>
+  <w:LsdException Locked="false" Priority="70" Name="Dark List Accent 1"/>
+  <w:LsdException Locked="false" Priority="71" Name="Colorful Shading Accent 1"/>
+  <w:LsdException Locked="false" Priority="72" Name="Colorful List Accent 1"/>
+  <w:LsdException Locked="false" Priority="73" Name="Colorful Grid Accent 1"/>
+  <w:LsdException Locked="false" Priority="60" Name="Light Shading Accent 2"/>
+  <w:LsdException Locked="false" Priority="61" Name="Light List Accent 2"/>
+  <w:LsdException Locked="false" Priority="62" Name="Light Grid Accent 2"/>
+  <w:LsdException Locked="false" Priority="63" Name="Medium Shading 1 Accent 2"/>
+  <w:LsdException Locked="false" Priority="64" Name="Medium Shading 2 Accent 2"/>
+  <w:LsdException Locked="false" Priority="65" Name="Medium List 1 Accent 2"/>
+  <w:LsdException Locked="false" Priority="66" Name="Medium List 2 Accent 2"/>
+  <w:LsdException Locked="false" Priority="67" Name="Medium Grid 1 Accent 2"/>
+  <w:LsdException Locked="false" Priority="68" Name="Medium Grid 2 Accent 2"/>
+  <w:LsdException Locked="false" Priority="69" Name="Medium Grid 3 Accent 2"/>
+  <w:LsdException Locked="false" Priority="70" Name="Dark List Accent 2"/>
+  <w:LsdException Locked="false" Priority="71" Name="Colorful Shading Accent 2"/>
+  <w:LsdException Locked="false" Priority="72" Name="Colorful List Accent 2"/>
+  <w:LsdException Locked="false" Priority="73" Name="Colorful Grid Accent 2"/>
+  <w:LsdException Locked="false" Priority="60" Name="Light Shading Accent 3"/>
+  <w:LsdException Locked="false" Priority="61" Name="Light List Accent 3"/>
+  <w:LsdException Locked="false" Priority="62" Name="Light Grid Accent 3"/>
+  <w:LsdException Locked="false" Priority="63" Name="Medium Shading 1 Accent 3"/>
+  <w:LsdException Locked="false" Priority="64" Name="Medium Shading 2 Accent 3"/>
+  <w:LsdException Locked="false" Priority="65" Name="Medium List 1 Accent 3"/>
+  <w:LsdException Locked="false" Priority="66" Name="Medium List 2 Accent 3"/>
+  <w:LsdException Locked="false" Priority="67" Name="Medium Grid 1 Accent 3"/>
+  <w:LsdException Locked="false" Priority="68" Name="Medium Grid 2 Accent 3"/>
+  <w:LsdException Locked="false" Priority="69" Name="Medium Grid 3 Accent 3"/>
+  <w:LsdException Locked="false" Priority="70" Name="Dark List Accent 3"/>
+  <w:LsdException Locked="false" Priority="71" Name="Colorful Shading Accent 3"/>
+  <w:LsdException Locked="false" Priority="72" Name="Colorful List Accent 3"/>
+  <w:LsdException Locked="false" Priority="73" Name="Colorful Grid Accent 3"/>
+  <w:LsdException Locked="false" Priority="60" Name="Light Shading Accent 4"/>
+  <w:LsdException Locked="false" Priority="61" Name="Light List Accent 4"/>
+  <w:LsdException Locked="false" Priority="62" Name="Light Grid Accent 4"/>
+  <w:LsdException Locked="false" Priority="63" Name="Medium Shading 1 Accent 4"/>
+  <w:LsdException Locked="false" Priority="64" Name="Medium Shading 2 Accent 4"/>
+  <w:LsdException Locked="false" Priority="65" Name="Medium List 1 Accent 4"/>
+  <w:LsdException Locked="false" Priority="66" Name="Medium List 2 Accent 4"/>
+  <w:LsdException Locked="false" Priority="67" Name="Medium Grid 1 Accent 4"/>
+  <w:LsdException Locked="false" Priority="68" Name="Medium Grid 2 Accent 4"/>
+  <w:LsdException Locked="false" Priority="69" Name="Medium Grid 3 Accent 4"/>
+  <w:LsdException Locked="false" Priority="70" Name="Dark List Accent 4"/>
+  <w:LsdException Locked="false" Priority="71" Name="Colorful Shading Accent 4"/>
+  <w:LsdException Locked="false" Priority="72" Name="Colorful List Accent 4"/>
+  <w:LsdException Locked="false" Priority="73" Name="Colorful Grid Accent 4"/>
+  <w:LsdException Locked="false" Priority="60" Name="Light Shading Accent 5"/>
+  <w:LsdException Locked="false" Priority="61" Name="Light List Accent 5"/>
+  <w:LsdException Locked="false" Priority="62" Name="Light Grid Accent 5"/>
+  <w:LsdException Locked="false" Priority="63" Name="Medium Shading 1 Accent 5"/>
+  <w:LsdException Locked="false" Priority="64" Name="Medium Shading 2 Accent 5"/>
+  <w:LsdException Locked="false" Priority="65" Name="Medium List 1 Accent 5"/>
+  <w:LsdException Locked="false" Priority="66" Name="Medium List 2 Accent 5"/>
+  <w:LsdException Locked="false" Priority="67" Name="Medium Grid 1 Accent 5"/>
+  <w:LsdException Locked="false" Priority="68" Name="Medium Grid 2 Accent 5"/>
+  <w:LsdException Locked="false" Priority="69" Name="Medium Grid 3 Accent 5"/>
+  <w:LsdException Locked="false" Priority="70" Name="Dark List Accent 5"/>
+  <w:LsdException Locked="false" Priority="71" Name="Colorful Shading Accent 5"/>
+  <w:LsdException Locked="false" Priority="72" Name="Colorful List Accent 5"/>
+  <w:LsdException Locked="false" Priority="73" Name="Colorful Grid Accent 5"/>
+  <w:LsdException Locked="false" Priority="60" Name="Light Shading Accent 6"/>
+  <w:LsdException Locked="false" Priority="61" Name="Light List Accent 6"/>
+  <w:LsdException Locked="false" Priority="62" Name="Light Grid Accent 6"/>
+  <w:LsdException Locked="false" Priority="63" Name="Medium Shading 1 Accent 6"/>
+  <w:LsdException Locked="false" Priority="64" Name="Medium Shading 2 Accent 6"/>
+  <w:LsdException Locked="false" Priority="65" Name="Medium List 1 Accent 6"/>
+  <w:LsdException Locked="false" Priority="66" Name="Medium List 2 Accent 6"/>
+  <w:LsdException Locked="false" Priority="67" Name="Medium Grid 1 Accent 6"/>
+  <w:LsdException Locked="false" Priority="68" Name="Medium Grid 2 Accent 6"/>
+  <w:LsdException Locked="false" Priority="69" Name="Medium Grid 3 Accent 6"/>
+  <w:LsdException Locked="false" Priority="70" Name="Dark List Accent 6"/>
+  <w:LsdException Locked="false" Priority="71" Name="Colorful Shading Accent 6"/>
+  <w:LsdException Locked="false" Priority="72" Name="Colorful List Accent 6"/>
+  <w:LsdException Locked="false" Priority="73" Name="Colorful Grid Accent 6"/>
+  <w:LsdException Locked="false" Priority="19" QFormat="true"
+   Name="Subtle Emphasis"/>
+  <w:LsdException Locked="false" Priority="21" QFormat="true"
+   Name="Intense Emphasis"/>
+  <w:LsdException Locked="false" Priority="31" QFormat="true"
+   Name="Subtle Reference"/>
+  <w:LsdException Locked="false" Priority="32" QFormat="true"
+   Name="Intense Reference"/>
+  <w:LsdException Locked="false" Priority="33" QFormat="true" Name="Book Title"/>
+  <w:LsdException Locked="false" Priority="37" SemiHidden="true"
+   UnhideWhenUsed="true" Name="Bibliography"/>
+  <w:LsdException Locked="false" Priority="39" SemiHidden="true"
+   UnhideWhenUsed="true" QFormat="true" Name="TOC Heading"/>
+  <w:LsdException Locked="false" Priority="41" Name="Plain Table 1"/>
+  <w:LsdException Locked="false" Priority="42" Name="Plain Table 2"/>
+  <w:LsdException Locked="false" Priority="43" Name="Plain Table 3"/>
+  <w:LsdException Locked="false" Priority="44" Name="Plain Table 4"/>
+  <w:LsdException Locked="false" Priority="45" Name="Plain Table 5"/>
+  <w:LsdException Locked="false" Priority="40" Name="Grid Table Light"/>
+  <w:LsdException Locked="false" Priority="46" Name="Grid Table 1 Light"/>
+  <w:LsdException Locked="false" Priority="47" Name="Grid Table 2"/>
+  <w:LsdException Locked="false" Priority="48" Name="Grid Table 3"/>
+  <w:LsdException Locked="false" Priority="49" Name="Grid Table 4"/>
+  <w:LsdException Locked="false" Priority="50" Name="Grid Table 5 Dark"/>
+  <w:LsdException Locked="false" Priority="51" Name="Grid Table 6 Colorful"/>
+  <w:LsdException Locked="false" Priority="52" Name="Grid Table 7 Colorful"/>
+  <w:LsdException Locked="false" Priority="46"
+   Name="Grid Table 1 Light Accent 1"/>
+  <w:LsdException Locked="false" Priority="47" Name="Grid Table 2 Accent 1"/>
+  <w:LsdException Locked="false" Priority="48" Name="Grid Table 3 Accent 1"/>
+  <w:LsdException Locked="false" Priority="49" Name="Grid Table 4 Accent 1"/>
+  <w:LsdException Locked="false" Priority="50" Name="Grid Table 5 Dark Accent 1"/>
+  <w:LsdException Locked="false" Priority="51"
+   Name="Grid Table 6 Colorful Accent 1"/>
+  <w:LsdException Locked="false" Priority="52"
+   Name="Grid Table 7 Colorful Accent 1"/>
+  <w:LsdException Locked="false" Priority="46"
+   Name="Grid Table 1 Light Accent 2"/>
+  <w:LsdException Locked="false" Priority="47" Name="Grid Table 2 Accent 2"/>
+  <w:LsdException Locked="false" Priority="48" Name="Grid Table 3 Accent 2"/>
+  <w:LsdException Locked="false" Priority="49" Name="Grid Table 4 Accent 2"/>
+  <w:LsdException Locked="false" Priority="50" Name="Grid Table 5 Dark Accent 2"/>
+  <w:LsdException Locked="false" Priority="51"
+   Name="Grid Table 6 Colorful Accent 2"/>
+  <w:LsdException Locked="false" Priority="52"
+   Name="Grid Table 7 Colorful Accent 2"/>
+  <w:LsdException Locked="false" Priority="46"
+   Name="Grid Table 1 Light Accent 3"/>
+  <w:LsdException Locked="false" Priority="47" Name="Grid Table 2 Accent 3"/>
+  <w:LsdException Locked="false" Priority="48" Name="Grid Table 3 Accent 3"/>
+  <w:LsdException Locked="false" Priority="49" Name="Grid Table 4 Accent 3"/>
+  <w:LsdException Locked="false" Priority="50" Name="Grid Table 5 Dark Accent 3"/>
+  <w:LsdException Locked="false" Priority="51"
+   Name="Grid Table 6 Colorful Accent 3"/>
+  <w:LsdException Locked="false" Priority="52"
+   Name="Grid Table 7 Colorful Accent 3"/>
+  <w:LsdException Locked="false" Priority="46"
+   Name="Grid Table 1 Light Accent 4"/>
+  <w:LsdException Locked="false" Priority="47" Name="Grid Table 2 Accent 4"/>
+  <w:LsdException Locked="false" Priority="48" Name="Grid Table 3 Accent 4"/>
+  <w:LsdException Locked="false" Priority="49" Name="Grid Table 4 Accent 4"/>
+  <w:LsdException Locked="false" Priority="50" Name="Grid Table 5 Dark Accent 4"/>
+  <w:LsdException Locked="false" Priority="51"
+   Name="Grid Table 6 Colorful Accent 4"/>
+  <w:LsdException Locked="false" Priority="52"
+   Name="Grid Table 7 Colorful Accent 4"/>
+  <w:LsdException Locked="false" Priority="46"
+   Name="Grid Table 1 Light Accent 5"/>
+  <w:LsdException Locked="false" Priority="47" Name="Grid Table 2 Accent 5"/>
+  <w:LsdException Locked="false" Priority="48" Name="Grid Table 3 Accent 5"/>
+  <w:LsdException Locked="false" Priority="49" Name="Grid Table 4 Accent 5"/>
+  <w:LsdException Locked="false" Priority="50" Name="Grid Table 5 Dark Accent 5"/>
+  <w:LsdException Locked="false" Priority="51"
+   Name="Grid Table 6 Colorful Accent 5"/>
+  <w:LsdException Locked="false" Priority="52"
+   Name="Grid Table 7 Colorful Accent 5"/>
+  <w:LsdException Locked="false" Priority="46"
+   Name="Grid Table 1 Light Accent 6"/>
+  <w:LsdException Locked="false" Priority="47" Name="Grid Table 2 Accent 6"/>
+  <w:LsdException Locked="false" Priority="48" Name="Grid Table 3 Accent 6"/>
+  <w:LsdException Locked="false" Priority="49" Name="Grid Table 4 Accent 6"/>
+  <w:LsdException Locked="false" Priority="50" Name="Grid Table 5 Dark Accent 6"/>
+  <w:LsdException Locked="false" Priority="51"
+   Name="Grid Table 6 Colorful Accent 6"/>
+  <w:LsdException Locked="false" Priority="52"
+   Name="Grid Table 7 Colorful Accent 6"/>
+  <w:LsdException Locked="false" Priority="46" Name="List Table 1 Light"/>
+  <w:LsdException Locked="false" Priority="47" Name="List Table 2"/>
+  <w:LsdException Locked="false" Priority="48" Name="List Table 3"/>
+  <w:LsdException Locked="false" Priority="49" Name="List Table 4"/>
+  <w:LsdException Locked="false" Priority="50" Name="List Table 5 Dark"/>
+  <w:LsdException Locked="false" Priority="51" Name="List Table 6 Colorful"/>
+  <w:LsdException Locked="false" Priority="52" Name="List Table 7 Colorful"/>
+  <w:LsdException Locked="false" Priority="46"
+   Name="List Table 1 Light Accent 1"/>
+  <w:LsdException Locked="false" Priority="47" Name="List Table 2 Accent 1"/>
+  <w:LsdException Locked="false" Priority="48" Name="List Table 3 Accent 1"/>
+  <w:LsdException Locked="false" Priority="49" Name="List Table 4 Accent 1"/>
+  <w:LsdException Locked="false" Priority="50" Name="List Table 5 Dark Accent 1"/>
+  <w:LsdException Locked="false" Priority="51"
+   Name="List Table 6 Colorful Accent 1"/>
+  <w:LsdException Locked="false" Priority="52"
+   Name="List Table 7 Colorful Accent 1"/>
+  <w:LsdException Locked="false" Priority="46"
+   Name="List Table 1 Light Accent 2"/>
+  <w:LsdException Locked="false" Priority="47" Name="List Table 2 Accent 2"/>
+  <w:LsdException Locked="false" Priority="48" Name="List Table 3 Accent 2"/>
+  <w:LsdException Locked="false" Priority="49" Name="List Table 4 Accent 2"/>
+  <w:LsdException Locked="false" Priority="50" Name="List Table 5 Dark Accent 2"/>
+  <w:LsdException Locked="false" Priority="51"
+   Name="List Table 6 Colorful Accent 2"/>
+  <w:LsdException Locked="false" Priority="52"
+   Name="List Table 7 Colorful Accent 2"/>
+  <w:LsdException Locked="false" Priority="46"
+   Name="List Table 1 Light Accent 3"/>
+  <w:LsdException Locked="false" Priority="47" Name="List Table 2 Accent 3"/>
+  <w:LsdException Locked="false" Priority="48" Name="List Table 3 Accent 3"/>
+  <w:LsdException Locked="false" Priority="49" Name="List Table 4 Accent 3"/>
+  <w:LsdException Locked="false" Priority="50" Name="List Table 5 Dark Accent 3"/>
+  <w:LsdException Locked="false" Priority="51"
+   Name="List Table 6 Colorful Accent 3"/>
+  <w:LsdException Locked="false" Priority="52"
+   Name="List Table 7 Colorful Accent 3"/>
+  <w:LsdException Locked="false" Priority="46"
+   Name="List Table 1 Light Accent 4"/>
+  <w:LsdException Locked="false" Priority="47" Name="List Table 2 Accent 4"/>
+  <w:LsdException Locked="false" Priority="48" Name="List Table 3 Accent 4"/>
+  <w:LsdException Locked="false" Priority="49" Name="List Table 4 Accent 4"/>
+  <w:LsdException Locked="false" Priority="50" Name="List Table 5 Dark Accent 4"/>
+  <w:LsdException Locked="false" Priority="51"
+   Name="List Table 6 Colorful Accent 4"/>
+  <w:LsdException Locked="false" Priority="52"
+   Name="List Table 7 Colorful Accent 4"/>
+  <w:LsdException Locked="false" Priority="46"
+   Name="List Table 1 Light Accent 5"/>
+  <w:LsdException Locked="false" Priority="47" Name="List Table 2 Accent 5"/>
+  <w:LsdException Locked="false" Priority="48" Name="List Table 3 Accent 5"/>
+  <w:LsdException Locked="false" Priority="49" Name="List Table 4 Accent 5"/>
+  <w:LsdException Locked="false" Priority="50" Name="List Table 5 Dark Accent 5"/>
+  <w:LsdException Locked="false" Priority="51"
+   Name="List Table 6 Colorful Accent 5"/>
+  <w:LsdException Locked="false" Priority="52"
+   Name="List Table 7 Colorful Accent 5"/>
+  <w:LsdException Locked="false" Priority="46"
+   Name="List Table 1 Light Accent 6"/>
+  <w:LsdException Locked="false" Priority="47" Name="List Table 2 Accent 6"/>
+  <w:LsdException Locked="false" Priority="48" Name="List Table 3 Accent 6"/>
+  <w:LsdException Locked="false" Priority="49" Name="List Table 4 Accent 6"/>
+  <w:LsdException Locked="false" Priority="50" Name="List Table 5 Dark Accent 6"/>
+  <w:LsdException Locked="false" Priority="51"
+   Name="List Table 6 Colorful Accent 6"/>
+  <w:LsdException Locked="false" Priority="52"
+   Name="List Table 7 Colorful Accent 6"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Mention"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Smart Hyperlink"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Hashtag"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Unresolved Mention"/>
+  <w:LsdException Locked="false" SemiHidden="true" UnhideWhenUsed="true"
+   Name="Smart Link"/>
+ </w:LatentStyles>
+</xml><![endif]-->
+<style>
+<!--
+ /* Font Definitions */
+ @font-face
+	{font-family:"Cambria Math";
+	panose-1:2 4 5 3 5 4 6 3 2 4;
+	mso-font-charset:0;
+	mso-generic-font-family:roman;
+	mso-font-pitch:variable;
+	mso-font-signature:-536869121 1107305727 33554432 0 415 0;}
+@font-face
+	{font-family:Calibri;
+	panose-1:2 15 5 2 2 2 4 3 2 4;
+	mso-font-charset:0;
+	mso-generic-font-family:swiss;
+	mso-font-pitch:variable;
+	mso-font-signature:-469750017 -1073732485 9 0 511 0;}
+ /* Style Definitions */
+ p.MsoNormal, li.MsoNormal, div.MsoNormal
+	{mso-style-unhide:no;
+	mso-style-qformat:yes;
+	mso-style-parent:"";
+	margin-top:0cm;
+	margin-right:0cm;
+	margin-bottom:8.0pt;
+	margin-left:0cm;
+	line-height:107%;
+	mso-pagination:widow-orphan;
+	font-size:11.0pt;
+	font-family:"Calibri",sans-serif;
+	mso-ascii-font-family:Calibri;
+	mso-ascii-theme-font:minor-latin;
+	mso-fareast-font-family:Calibri;
+	mso-fareast-theme-font:minor-latin;
+	mso-hansi-font-family:Calibri;
+	mso-hansi-theme-font:minor-latin;
+	mso-bidi-font-family:"Times New Roman";
+	mso-bidi-theme-font:minor-bidi;
+	mso-font-kerning:1.0pt;
+	mso-ligatures:standardcontextual;
+	mso-fareast-language:EN-US;}
+.MsoChpDefault
+	{mso-style-type:export-only;
+	mso-default-props:yes;
+	font-family:"Calibri",sans-serif;
+	mso-ascii-font-family:Calibri;
+	mso-ascii-theme-font:minor-latin;
+	mso-fareast-font-family:Calibri;
+	mso-fareast-theme-font:minor-latin;
+	mso-hansi-font-family:Calibri;
+	mso-hansi-theme-font:minor-latin;
+	mso-bidi-font-family:"Times New Roman";
+	mso-bidi-theme-font:minor-bidi;
+	mso-fareast-language:EN-US;}
+.MsoPapDefault
+	{mso-style-type:export-only;
+	margin-bottom:8.0pt;
+	line-height:107%;}
+@page WordSection1
+	{size:595.3pt 841.9pt;
+	margin:72.0pt 72.0pt 72.0pt 72.0pt;
+	mso-header-margin:35.4pt;
+	mso-footer-margin:35.4pt;
+	mso-paper-source:0;}
+div.WordSection1
+	{page:WordSection1;}
+-->
+</style>
+<!--[if gte mso 10]>
+<style>
+ /* Style Definitions */
+ table.MsoNormalTable
+	{mso-style-name:"Table Normal";
+	mso-tstyle-rowband-size:0;
+	mso-tstyle-colband-size:0;
+	mso-style-noshow:yes;
+	mso-style-priority:99;
+	mso-style-parent:"";
+	mso-padding-alt:0cm 5.4pt 0cm 5.4pt;
+	mso-para-margin-top:0cm;
+	mso-para-margin-right:0cm;
+	mso-para-margin-bottom:8.0pt;
+	mso-para-margin-left:0cm;
+	line-height:107%;
+	mso-pagination:widow-orphan;
+	font-size:11.0pt;
+	font-family:"Calibri",sans-serif;
+	mso-ascii-font-family:Calibri;
+	mso-ascii-theme-font:minor-latin;
+	mso-hansi-font-family:Calibri;
+	mso-hansi-theme-font:minor-latin;
+	mso-bidi-font-family:"Times New Roman";
+	mso-bidi-theme-font:minor-bidi;
+	mso-font-kerning:1.0pt;
+	mso-ligatures:standardcontextual;
+	mso-fareast-language:EN-US;}
+</style>
+<![endif]--><!--[if gte mso 9]><xml>
+ <o:shapedefaults v:ext="edit" spidmax="1026"/>
+</xml><![endif]--><!--[if gte mso 9]><xml>
+ <o:shapelayout v:ext="edit">
+  <o:idmap v:ext="edit" data="1"/>
+ </o:shapelayout></xml><![endif]-->
+</head>
+
+<body lang=EN-GB style='tab-interval:36.0pt;word-wrap:break-word'>
+
+<div class=WordSection1>
+
+<p class=MsoNormal>Hello, World!</p>
+
+</div>
+
+</body>
+
+</html>

test/sample.txt

@@ -0,0 +1 @@
+Hello, World!

test/sample_files/colorschememapping.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<a:clrMap xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" bg1="lt1" tx1="dk1" bg2="lt2" tx2="dk2" accent1="accent1" accent2="accent2" accent3="accent3" accent4="accent4" accent5="accent5" accent6="accent6" hlink="hlink" folHlink="folHlink"/>

test/sample_files/filelist.xml

@@ -0,0 +1,6 @@
+<xml xmlns:o="urn:schemas-microsoft-com:office:office">
+ <o:MainFile HRef="../sample.html"/>
+ <o:File HRef="themedata.thmx"/>
+ <o:File HRef="colorschememapping.xml"/>
+ <o:File HRef="filelist.xml"/>
+</xml>

test/test_module.py

@@ -0,0 +1,45 @@
+# ---
+# jupyter:
+#   jupytext:
+#     formats: ipynb,py:light
+#     text_representation:
+#       extension: .py
+#       format_name: light
+#       format_version: '1.5'
+#       jupytext_version: 1.15.0
+#   kernelspec:
+#     display_name: Python 3 (ipykernel)
+#     language: python
+#     name: python3
+# ---
+
+# +
+import pytest
+import gradio as gr
+from ..chatfuncs.ingest import *
+from ..chatfuncs.chatfuncs import *
+
+def test_read_docx():
+    content = read_docx('sample.docx')
+    assert content == "Hello, World!"
+
+
+# +
+def test_parse_file():
+    # Assuming these files exist and you know their content
+    files = ['sample.docx', 'sample.pdf', 'sample.txt', 'sample.html']
+    contents = parse_file(files)
+    
+    assert contents['sample.docx'] == 'Hello, World!'
+    assert contents['sample.pdf'] == 'Hello, World!'
+    assert contents['sample.txt'] == 'Hello, World!'
+    assert contents['sample.html'] == 'Hello, World!'
+
+def test_unsupported_file_type():
+    files = ['sample.unknown']
+    contents = parse_file(files)
+    assert contents['sample.unknown'].startswith('Unsupported file type:')
+
+def test_input_validation():
+    with pytest.raises(ValueError, match="Expected a list of file paths."):
+        parse_file('single_file_path.txt')