Update app.py

app.py

@@ -3,16 +3,15 @@ import os
 import time
 import pandas as pd
 import sqlite3
-import ocrmypdf
 import logging
 
-from langchain.document_loaders import OnlinePDFLoader  # for loading the PDF
+from langchain.document_loaders import OnlinePDFLoader  # for loading the PDF text
 from langchain.embeddings import HuggingFaceEmbeddings  # open source embedding model
 from langchain.text_splitter import CharacterTextSplitter
 from langchain_community.vectorstores import Chroma  # updated import for vectorization
 from langchain.chains import RetrievalQA  # for QA chain
 from langchain_community.chat_models import ChatOpenAI  # updated import for ChatOpenAI
-from langchain_core.prompts import PromptTemplate  # updated import per deprecation notice
+from langchain_core.prompts import PromptTemplate  # prompt template import
 
 # Setup basic logging
 logging.basicConfig(level=logging.INFO)
@@ -24,36 +23,16 @@ def update_log(message):
     log_messages += message + "\n"
     logger.info(message)
 
-def ocr_converter(input_file):
-    image_pdf = input_file.name
-    try:
-        # Disable deskew, clean_final, and remove_background to avoid compatibility issues with --redo-ocr.
-        ocrmypdf.ocr(
-            image_pdf,
-            image_pdf,
-            redo_ocr=True,
-            force_ocr=True,
-            language="eng",
-            output_type="pdf",
-            deskew=False,
-            clean_final=False,
-            remove_background=False
-        )
-        update_log(f"OCR conversion successful for {image_pdf}")
-    except Exception as e:
-        error_msg = f"OCR conversion failed for {image_pdf}. Error: {str(e)}"
-        update_log(error_msg)
-        raise e
-    return image_pdf
-
 def load_pdf_and_generate_embeddings(pdf_doc, open_ai_key, relevant_pages):
     try:
         if open_ai_key is not None:
             os.environ['OPENAI_API_KEY'] = open_ai_key
-        pdf_doc = ocr_converter(pdf_doc)
-        loader = OnlinePDFLoader(pdf_doc)
+        
+        # Use the file path directly as OCR is removed; text is extracted via the document loader.
+        pdf_path = pdf_doc.name
+        loader = OnlinePDFLoader(pdf_path)
         pages = loader.load_and_split()
-        update_log(f"Loaded {len(pages)} pages from {pdf_doc}")
+        update_log(f"Extracted text from {len(pages)} pages in {pdf_path}")
         
         embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
         pages_to_be_loaded = []
@@ -138,13 +117,13 @@ def load_master_questionset_into_sqlite(connection):
     update_log(f"Total questions in DB: {total_questions}")
 
 def create_field_and_question_list_for_DOC_A():
-    # Two sample entries
+    # Two sample entries for DOC_A
     fields = ["Loan Number", "Borrower"]
     queries = ["What is the Loan Number?", "Who is the Borrower?"]
     return fields, queries
 
 def create_field_and_question_list_for_DOC_B():
-    # Two sample entries
+    # Two sample entries for DOC_B
     fields = ["Property Address", "Signed Date"]
     queries = ["What is the Property Address?", "What is the Signed Date?"]
     return fields, queries
@@ -294,11 +273,26 @@ with gr.Blocks(css=css, theme=gr.themes.Monochrome()) as demo:
             log_window = gr.Textbox(label="Log Window", interactive=False, lines=10)
     
     with gr.Tab("OCR Converter"):
+        # This tab is now repurposed (or can be removed)
         with gr.Column():
-            image_pdf = gr.File(label="Load PDF for OCR", file_types=['.pdf'], type='filepath')
+            image_pdf = gr.File(label="Load PDF for Conversion", file_types=['.pdf'], type='filepath')
         with gr.Row():
-            ocr_pdf = gr.File(label="OCR'd PDF", file_types=['.pdf'], type='filepath', file_count="single")
-            convert_btn = gr.Button("Convert")
+            extracted_text = gr.Textbox(label="Extracted Text", lines=10)
+            extract_btn = gr.Button("Extract Text")
+    
+        # For demonstration, extract text using OnlinePDFLoader
+        def extract_text(pdf_file):
+            try:
+                loader = OnlinePDFLoader(pdf_file.name)
+                docs = loader.load_and_split()
+                text = "\n".join([doc.page_content for doc in docs])
+                update_log(f"Extracted text from {len(docs)} pages.")
+                return text
+            except Exception as e:
+                err = f"Error extracting text: {str(e)}"
+                update_log(err)
+                return err
+        extract_btn.click(extract_text, inputs=image_pdf, outputs=extracted_text)
     
     with gr.Tab("Upload Question Set"):
         with gr.Column():
@@ -320,7 +314,6 @@ with gr.Blocks(css=css, theme=gr.themes.Monochrome()) as demo:
     load_fields_btn.click(retrieve_fields_and_questions, inputs=questionsets, outputs=fields_and_questions)
     answer_predefined_btn.click(answer_predefined_questions, inputs=questionsets, outputs=answers_df)
     
-    convert_btn.click(ocr_converter, inputs=image_pdf, outputs=ocr_pdf)
     load_csv_btn.click(load_csv_and_store_questionset_into_sqlite, inputs=[csv_file, document_type_for_questionset, tag_for_questionset], outputs=status_for_csv)
 
 demo.launch(debug=True)