feat(check token size of context)

* fn that returns boolean for if token size is under 8000
* raise gr.Error if file exceeds token size

app.py

@@ -189,6 +189,12 @@ def model_call(question, document, answer, client_base_url):
     combined_reasoning = " ".join(reasoning)[1:-1]
     return combined_reasoning, score
 
+def return_approximate_token_size(text):
+    MAX_TOKEN_LENGTH = 8000
+    number_of_total_characters = len(text)
+    number_of_tokens = number_of_total_characters / 4
+    return number_of_tokens < MAX_TOKEN_LENGTH
+
 def get_filetype(filename):
     return filename.split(".")[-1]
 
@@ -218,6 +224,11 @@ def upload_file(filepath):
             extracted_file_text = extract_text_pymupdf(filepath)
         elif filetype == "docx":
             extracted_file_text = extract_text_python_docx(filepath)
+        
+        # return warning if file is too large
+        if not return_approximate_token_size(extracted_file_text):
+            raise gr.Error("File is too large to process. Please upload a smaller file.")
+
         return [gr.UploadButton(visible=False), gr.Group(visible=True), gr.Markdown(f"**Uploaded file:** {name}"), extracted_file_text]
     else:
         return [gr.UploadButton(visible=True, file_count="single", file_types=UPLOADABLE_FILE_TYPES), gr.Group(visible=False), gr.Markdown(""), extracted_file_text]