first commit

app.py

@@ -0,0 +1,129 @@
+import streamlit as st
+
+from knowledge_gpt.components.sidebar import sidebar
+
+from knowledge_gpt.ui import (
+    wrap_doc_in_html,
+    is_query_valid,
+    is_file_valid,
+    is_open_ai_key_valid,
+    display_file_read_error,
+)
+
+from knowledge_gpt.core.caching import bootstrap_caching
+
+from knowledge_gpt.core.parsing import read_file
+from knowledge_gpt.core.chunking import chunk_file
+from knowledge_gpt.core.embedding import embed_files
+from knowledge_gpt.core.qa import query_folder
+from knowledge_gpt.core.utils import get_llm
+
+
+EMBEDDING = "openai"
+VECTOR_STORE = "faiss"
+MODEL_LIST = ["gpt-3.5-turbo", "gpt-4"]
+
+# Uncomment to enable debug mode
+# MODEL_LIST.insert(0, "debug")
+
+st.set_page_config(page_title="ReportIO", page_icon="☘️", layout="wide")
+st.header("☘️ReportIO")
+
+# Enable caching for expensive functions
+bootstrap_caching()
+
+sidebar()
+
+openai_api_key = st.session_state.get("OPENAI_API_KEY")
+
+
+if not openai_api_key:
+    st.warning(
+        "Enter your OpenAI API key in the sidebar. You can get a key at"
+        " https://platform.openai.com/account/api-keys."
+    )
+
+
+uploaded_file = st.file_uploader(
+    "Upload a pdf, docx, or txt file",
+    type=["pdf", "docx", "txt"],
+    help="Scanned documents are not supported yet!",
+)
+
+# model: str = st.selectbox("Model", options=MODEL_LIST)  # type: ignore
+
+model = MODEL_LIST[0]
+
+
+
+with st.expander("Advanced Options"):
+    return_all_chunks = st.checkbox("Show all chunks retrieved from vector search")
+    show_full_doc = st.checkbox("Show parsed contents of the document")
+
+
+if not uploaded_file:
+    st.stop()
+
+try:
+    file = read_file(uploaded_file)
+except Exception as e:
+    display_file_read_error(e, file_name=uploaded_file.name)
+
+chunked_file = chunk_file(file, chunk_size=300, chunk_overlap=0)
+
+if not is_file_valid(file):
+    st.stop()
+
+
+if not is_open_ai_key_valid(openai_api_key, model):
+    st.stop()
+
+
+with st.spinner("Indexing document... This may take a while⏳"):
+    folder_index = embed_files(
+        files=[chunked_file],
+        embedding=EMBEDDING if model != "debug" else "debug",
+        vector_store=VECTOR_STORE if model != "debug" else "debug",
+        openai_api_key=openai_api_key,
+    )
+
+
+with st.form(key="qa_form"):
+    options = ['List all pre existing conditions which may affect home insurance', 'Show the problematic components!', 'Show repair needs!']
+    query = st.selectbox('Select an option', options)
+    submit = st.form_submit_button("Submit")
+
+
+if show_full_doc:
+    with st.expander("Document"):
+        # Hack to get around st.markdown rendering LaTeX
+        st.markdown(f"<p>{wrap_doc_in_html(file.docs)}</p>", unsafe_allow_html=True)
+
+
+if submit:
+    if not is_query_valid(query):
+        st.stop()
+
+    # Output Columns
+    answer_col, sources_col = st.columns(2)
+
+    llm = get_llm(model=model, openai_api_key=openai_api_key, temperature=0)
+    result = query_folder(
+        folder_index=folder_index,
+        query=query,
+        return_all=return_all_chunks,
+        llm=llm,
+    )
+
+    with answer_col:
+        st.markdown("#### Answer")
+        st.markdown(result.answer)
+
+    with sources_col:
+        st.markdown("#### Sources")
+        for source in result.sources:
+            st.markdown(source.page_content)
+            st.markdown(source.metadata["source"])
+            st.markdown("---")
+
+

components/faq.py

@@ -0,0 +1,39 @@
+# flake8: noqa
+import streamlit as st
+
+
+def faq():
+    st.markdown(
+        """
+# FAQ
+## How does ReportIO work?
+When you upload a document, it will be divided into smaller chunks 
+and stored in a special type of database called a vector index 
+that allows for semantic search and retrieval.
+
+When you ask a question, ReportIO will search through the
+document chunks and find the most relevant ones using the vector index.
+Then, it will use GPT3 to generate a final answer.
+
+## Is my data safe?
+Yes, your data is safe. ReportIO does not store your documents or
+questions. All uploaded data is deleted after you close the browser tab.
+
+## Why does it take so long to index my document?
+If you are using a free OpenAI API key, it will take a while to index
+your document. This is because the free API key has strict [rate limits](https://platform.openai.com/docs/guides/rate-limits/overview).
+To speed up the indexing process, you can use a paid API key.
+
+## What do the numbers mean under each source?
+For a PDF document, you will see a citation number like this: 3-12. 
+The first number is the page number and the second number is 
+the chunk number on that page. For DOCS and TXT documents, 
+the first number is set to 1 and the second number is the chunk number.
+
+## Are the answers 100% accurate?
+
+But for most use cases, ReportIO is very accurate and can answer
+most questions. Always check with the sources to make sure that the answers
+are correct.
+"""
+    )

components/sidebar.py

@@ -0,0 +1,42 @@
+import streamlit as st
+
+from knowledge_gpt.components.faq import faq
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+
+
+def sidebar():
+    with st.sidebar:
+        st.markdown(
+            "## How can I help you?\n"
+           
+            "1. Upload a pdf, docx, or txt file of home inspection report📄\n"
+            "2. Ask a question about the report\n"
+            "2. Or use existing extractor button to see analyzes. ⭐\n"
+        )
+        api_key_input = st.text_input(
+            "OpenAI API Key",
+            type="password",
+            placeholder="Paste your OpenAI API key here (sk-...)",
+            help="You can get your API key from https://platform.openai.com/account/api-keys.",  # noqa: E501
+            value=os.environ.get("OPENAI_API_KEY", None)
+            or st.session_state.get("OPENAI_API_KEY", ""),
+        )
+        
+        st.session_state["OPENAI_API_KEY"] = api_key_input
+
+        st.markdown("---")
+        st.markdown("# About")
+        st.markdown(
+            "☘️ReportIO allows you to ask questions about your "
+            "home inspection reports and get accurate answers with instant citations. "
+        )
+        st.markdown(
+            "This tool is a work in progress. "
+        )
+        st.markdown("Made by S.Atan")
+        st.markdown("---")
+
+        faq()

core/caching.py

@@ -0,0 +1,33 @@
+import streamlit as st
+from streamlit.runtime.caching.hashing import HashFuncsDict
+
+import knowledge_gpt.core.parsing as parsing
+import knowledge_gpt.core.chunking as chunking
+import knowledge_gpt.core.embedding as embedding
+from knowledge_gpt.core.parsing import File
+
+
+def file_hash_func(file: File) -> str:
+    """Get a unique hash for a file"""
+    return file.id
+
+
+@st.cache_data(show_spinner=False)
+def bootstrap_caching():
+    """Patch module functions with caching"""
+
+    # Get all substypes of File from module
+    file_subtypes = [
+        cls
+        for cls in vars(parsing).values()
+        if isinstance(cls, type) and issubclass(cls, File) and cls != File
+    ]
+    file_hash_funcs: HashFuncsDict = {cls: file_hash_func for cls in file_subtypes}
+
+    parsing.read_file = st.cache_data(show_spinner=False)(parsing.read_file)
+    chunking.chunk_file = st.cache_data(show_spinner=False, hash_funcs=file_hash_funcs)(
+        chunking.chunk_file
+    )
+    embedding.embed_files = st.cache_data(
+        show_spinner=False, hash_funcs=file_hash_funcs
+    )(embedding.embed_files)

core/chunking.py

@@ -0,0 +1,38 @@
+from langchain.docstore.document import Document
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from knowledge_gpt.core.parsing import File
+
+
+def chunk_file(
+    file: File, chunk_size: int, chunk_overlap: int = 0, model_name="gpt-3.5-turbo"
+) -> File:
+    """Chunks each document in a file into smaller documents
+    according to the specified chunk size and overlap
+    where the size is determined by the number of tokens for the specified model.
+    """
+
+    # split each document into chunks
+    chunked_docs = []
+    for doc in file.docs:
+        text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
+            model_name=model_name,
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+        )
+
+        chunks = text_splitter.split_text(doc.page_content)
+
+        for i, chunk in enumerate(chunks):
+            doc = Document(
+                page_content=chunk,
+                metadata={
+                    "page": doc.metadata.get("page", 1),
+                    "chunk": i + 1,
+                    "source": f"{doc.metadata.get('page', 1)}-{i + 1}",
+                },
+            )
+            chunked_docs.append(doc)
+
+    chunked_file = file.copy()
+    chunked_file.docs = chunked_docs
+    return chunked_file

core/debug.py

@@ -0,0 +1,49 @@
+from langchain.vectorstores import VectorStore
+from typing import Iterable, List, Any
+from langchain.docstore.document import Document
+from langchain.embeddings.base import Embeddings
+from langchain.embeddings.fake import FakeEmbeddings as FakeEmbeddingsBase
+from langchain.chat_models.fake import FakeListChatModel
+from typing import Optional
+
+
+class FakeChatModel(FakeListChatModel):
+    def __init__(self, **kwargs):
+        responses = ["The answer is 42. SOURCES: 1, 2, 3, 4"]
+        super().__init__(responses=responses, **kwargs)
+
+
+class FakeEmbeddings(FakeEmbeddingsBase):
+    def __init__(self, **kwargs):
+        super().__init__(size=4, **kwargs)
+
+
+class FakeVectorStore(VectorStore):
+    """Fake vector store for testing purposes."""
+
+    def __init__(self, texts: List[str]):
+        self.texts: List[str] = texts
+
+    def add_texts(
+        self, texts: Iterable[str], metadatas: List[dict] | None = None, **kwargs: Any
+    ) -> List[str]:
+        self.texts.extend(texts)
+        return self.texts
+
+    @classmethod
+    def from_texts(
+        cls,
+        texts: List[str],
+        embedding: Embeddings,
+        metadatas: Optional[List[dict]] = None,
+        **kwargs: Any,
+    ) -> "FakeVectorStore":
+        return cls(texts=list(texts))
+
+    def similarity_search(
+        self, query: str, k: int = 4, **kwargs: Any
+    ) -> List[Document]:
+        return [
+            Document(page_content=text, metadata={"source": f"{i+1}-{1}"})
+            for i, text in enumerate(self.texts)
+        ]

core/embedding.py

@@ -0,0 +1,74 @@
+from langchain.vectorstores import VectorStore
+from knowledge_gpt.core.parsing import File
+from langchain.vectorstores.faiss import FAISS
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.embeddings.base import Embeddings
+from typing import List, Type
+from langchain.docstore.document import Document
+from knowledge_gpt.core.debug import FakeVectorStore, FakeEmbeddings
+
+
+class FolderIndex:
+    """Index for a collection of files (a folder)"""
+
+    def __init__(self, files: List[File], index: VectorStore):
+        self.name: str = "default"
+        self.files = files
+        self.index: VectorStore = index
+
+    @staticmethod
+    def _combine_files(files: List[File]) -> List[Document]:
+        """Combines all the documents in a list of files into a single list."""
+
+        all_texts = []
+        for file in files:
+            for doc in file.docs:
+                doc.metadata["file_name"] = file.name
+                doc.metadata["file_id"] = file.id
+                all_texts.append(doc)
+
+        return all_texts
+
+    @classmethod
+    def from_files(
+        cls, files: List[File], embeddings: Embeddings, vector_store: Type[VectorStore]
+    ) -> "FolderIndex":
+        """Creates an index from files."""
+
+        all_docs = cls._combine_files(files)
+
+        index = vector_store.from_documents(
+            documents=all_docs,
+            embedding=embeddings,
+        )
+
+        return cls(files=files, index=index)
+
+
+def embed_files(
+    files: List[File], embedding: str, vector_store: str, **kwargs
+) -> FolderIndex:
+    """Embeds a collection of files and stores them in a FolderIndex."""
+
+    supported_embeddings: dict[str, Type[Embeddings]] = {
+        "openai": OpenAIEmbeddings,
+        "debug": FakeEmbeddings,
+    }
+    supported_vector_stores: dict[str, Type[VectorStore]] = {
+        "faiss": FAISS,
+        "debug": FakeVectorStore,
+    }
+
+    if embedding in supported_embeddings:
+        _embeddings = supported_embeddings[embedding](**kwargs)
+    else:
+        raise NotImplementedError(f"Embedding {embedding} not supported.")
+
+    if vector_store in supported_vector_stores:
+        _vector_store = supported_vector_stores[vector_store]
+    else:
+        raise NotImplementedError(f"Vector store {vector_store} not supported.")
+
+    return FolderIndex.from_files(
+        files=files, embeddings=_embeddings, vector_store=_vector_store
+    )

core/parsing.py

@@ -0,0 +1,108 @@
+from io import BytesIO
+from typing import List, Any, Optional
+import re
+
+import docx2txt
+from langchain.docstore.document import Document
+import fitz
+from hashlib import md5
+
+from abc import abstractmethod, ABC
+from copy import deepcopy
+
+
+class File(ABC):
+    """Represents an uploaded file comprised of Documents"""
+
+    def __init__(
+        self,
+        name: str,
+        id: str,
+        metadata: Optional[dict[str, Any]] = None,
+        docs: Optional[List[Document]] = None,
+    ):
+        self.name = name
+        self.id = id
+        self.metadata = metadata or {}
+        self.docs = docs or []
+
+    @classmethod
+    @abstractmethod
+    def from_bytes(cls, file: BytesIO) -> "File":
+        """Creates a File from a BytesIO object"""
+
+    def __repr__(self) -> str:
+        return (
+            f"File(name={self.name}, id={self.id},"
+            " metadata={self.metadata}, docs={self.docs})"
+        )
+
+    def __str__(self) -> str:
+        return f"File(name={self.name}, id={self.id}, metadata={self.metadata})"
+
+    def copy(self) -> "File":
+        """Create a deep copy of this File"""
+        return self.__class__(
+            name=self.name,
+            id=self.id,
+            metadata=deepcopy(self.metadata),
+            docs=deepcopy(self.docs),
+        )
+
+
+def strip_consecutive_newlines(text: str) -> str:
+    """Strips consecutive newlines from a string
+    possibly with whitespace in between
+    """
+    return re.sub(r"\s*\n\s*", "\n", text)
+
+
+class DocxFile(File):
+    @classmethod
+    def from_bytes(cls, file: BytesIO) -> "DocxFile":
+        text = docx2txt.process(file)
+        text = strip_consecutive_newlines(text)
+        doc = Document(page_content=text.strip())
+        doc.metadata["source"] = "p-1"
+        return cls(name=file.name, id=md5(file.read()).hexdigest(), docs=[doc])
+
+
+class PdfFile(File):
+    @classmethod
+    def from_bytes(cls, file: BytesIO) -> "PdfFile":
+        pdf = fitz.open(stream=file.read(), filetype="pdf")  # type: ignore
+        docs = []
+        for i, page in enumerate(pdf):
+            text = page.get_text(sort=True)
+            text = strip_consecutive_newlines(text)
+            doc = Document(page_content=text.strip())
+            doc.metadata["page"] = i + 1
+            doc.metadata["source"] = f"p-{i+1}"
+            docs.append(doc)
+        # file.read() mutates the file object, which can affect caching
+        # so we need to reset the file pointer to the beginning
+        file.seek(0)
+        return cls(name=file.name, id=md5(file.read()).hexdigest(), docs=docs)
+
+
+class TxtFile(File):
+    @classmethod
+    def from_bytes(cls, file: BytesIO) -> "TxtFile":
+        text = file.read().decode("utf-8", errors="replace")
+        text = strip_consecutive_newlines(text)
+        file.seek(0)
+        doc = Document(page_content=text.strip())
+        doc.metadata["source"] = "p-1"
+        return cls(name=file.name, id=md5(file.read()).hexdigest(), docs=[doc])
+
+
+def read_file(file: BytesIO) -> File:
+    """Reads an uploaded file and returns a File object"""
+    if file.name.lower().endswith(".docx"):
+        return DocxFile.from_bytes(file)
+    elif file.name.lower().endswith(".pdf"):
+        return PdfFile.from_bytes(file)
+    elif file.name.lower().endswith(".txt"):
+        return TxtFile.from_bytes(file)
+    else:
+        raise NotImplementedError(f"File type {file.name.split('.')[-1]} not supported")

core/prompts.py

@@ -0,0 +1,31 @@
+# flake8: noqa
+from langchain.prompts import PromptTemplate
+
+## Use a shorter template to reduce the number of tokens in the prompt
+template = """Create a final answer to the given questions using the provided document excerpts (given in no particular order) as sources. ALWAYS include a "SOURCES" section in your answer citing only the minimal set of sources needed to answer the question. If you are unable to answer the question, simply state that you do not have enough information to answer the question and leave the SOURCES section empty. Use only the provided documents and do not attempt to fabricate an answer.
+
+---------
+
+QUESTION: What  is the purpose of ARPA-H?
+=========
+Content: More support for patients and families. \n\nTo get there, I call on Congress to fund ARPA-H, the Advanced Research Projects Agency for Health. \n\nIt's based on DARPA—the Defense Department project that led to the Internet, GPS, and so much more.  \n\nARPA-H will have a singular purpose—to drive breakthroughs in cancer, Alzheimer's, diabetes, and more.
+SOURCES: 1-32
+Content: While we're at it, let's make sure every American can get the health care they need. \n\nWe've already made historic investments in health care. \n\nWe've made it easier for Americans to get the care they need, when they need it. \n\nWe've made it easier for Americans to get the treatments they need, when they need them. \n\nWe've made it easier for Americans to get the medications they need, when they need them.
+SOURCES: 1-33
+Content: The V.A. is pioneering new ways of linking toxic exposures to disease, already helping  veterans get the care they deserve. \n\nWe need to extend that same care to all Americans. \n\nThat's why I'm calling on Congress to pass legislation that would establish a national registry of toxic exposures, and provide health care and financial assistance to those affected.
+SOURCES: 1-30
+=========
+FINAL ANSWER: The purpose of ARPA-H is to drive breakthroughs in cancer, Alzheimer's, diabetes, and more.
+SOURCES: 1-32
+
+---------
+
+QUESTION: {question}
+=========
+{summaries}
+=========
+FINAL ANSWER:"""
+
+STUFF_PROMPT = PromptTemplate(
+    template=template, input_variables=["summaries", "question"]
+)

core/qa.py

@@ -0,0 +1,65 @@
+from typing import List
+from langchain.chains.qa_with_sources import load_qa_with_sources_chain
+from knowledge_gpt.core.prompts import STUFF_PROMPT
+from langchain.docstore.document import Document
+from knowledge_gpt.core.embedding import FolderIndex
+from pydantic import BaseModel
+from langchain.chat_models.base import BaseChatModel
+
+
+class AnswerWithSources(BaseModel):
+    answer: str
+    sources: List[Document]
+
+
+def query_folder(
+    query: str,
+    folder_index: FolderIndex,
+    llm: BaseChatModel,
+    return_all: bool = False,
+) -> AnswerWithSources:
+    """Queries a folder index for an answer.
+
+    Args:
+        query (str): The query to search for.
+        folder_index (FolderIndex): The folder index to search.
+        return_all (bool): Whether to return all the documents from the embedding or
+        just the sources for the answer.
+        model (str): The model to use for the answer generation.
+        **model_kwargs (Any): Keyword arguments for the model.
+
+    Returns:
+        AnswerWithSources: The answer and the source documents.
+    """
+
+    chain = load_qa_with_sources_chain(
+        llm=llm,
+        chain_type="stuff",
+        prompt=STUFF_PROMPT,
+    )
+
+    relevant_docs = folder_index.index.similarity_search(query, k=5)
+    result = chain(
+        {"input_documents": relevant_docs, "question": query}, return_only_outputs=True
+    )
+    sources = relevant_docs
+
+    if not return_all:
+        sources = get_sources(result["output_text"], folder_index)
+
+    answer = result["output_text"].split("SOURCES: ")[0]
+
+    return AnswerWithSources(answer=answer, sources=sources)
+
+
+def get_sources(answer: str, folder_index: FolderIndex) -> List[Document]:
+    """Retrieves the docs that were used to answer the question the generated answer."""
+
+    source_keys = [s for s in answer.split("SOURCES: ")[-1].split(", ")]
+
+    source_docs = []
+    for file in folder_index.files:
+        for doc in file.docs:
+            if doc.metadata["source"] in source_keys:
+                source_docs.append(doc)
+    return source_docs

core/utils.py

@@ -0,0 +1,32 @@
+from typing import List
+from langchain.chains.combine_documents.stuff import StuffDocumentsChain
+from langchain.docstore.document import Document
+
+from langchain.chat_models import ChatOpenAI
+from knowledge_gpt.core.debug import FakeChatModel
+from langchain.chat_models.base import BaseChatModel
+
+
+def pop_docs_upto_limit(
+    query: str, chain: StuffDocumentsChain, docs: List[Document], max_len: int
+) -> List[Document]:
+    """Pops documents from a list until the final prompt length is less
+    than the max length."""
+
+    token_count: int = chain.prompt_length(docs, question=query)  # type: ignore
+
+    while token_count > max_len and len(docs) > 0:
+        docs.pop()
+        token_count = chain.prompt_length(docs, question=query)  # type: ignore
+
+    return docs
+
+
+def get_llm(model: str, **kwargs) -> BaseChatModel:
+    if model == "debug":
+        return FakeChatModel()
+
+    if "gpt" in model:
+        return ChatOpenAI(model=model, **kwargs)  # type: ignore
+
+    raise NotImplementedError(f"Model {model} not supported!")

ui.py

@@ -0,0 +1,64 @@
+from typing import List
+import streamlit as st
+from langchain.docstore.document import Document
+from knowledge_gpt.core.parsing import File
+import openai
+from streamlit.logger import get_logger
+from typing import NoReturn
+
+logger = get_logger(__name__)
+
+
+def wrap_doc_in_html(docs: List[Document]) -> str:
+    """Wraps each page in document separated by newlines in <p> tags"""
+    text = [doc.page_content for doc in docs]
+    if isinstance(text, list):
+        # Add horizontal rules between pages
+        text = "\n<hr/>\n".join(text)
+    return "".join([f"<p>{line}</p>" for line in text.split("\n")])
+
+
+def is_query_valid(query: str) -> bool:
+    if not query:
+        st.error("Please enter a question!")
+        return False
+    return True
+
+
+def is_file_valid(file: File) -> bool:
+    if (
+        len(file.docs) == 0
+        or "".join([doc.page_content for doc in file.docs]).strip() == ""
+    ):
+        st.error("Cannot read document! Make sure the document has selectable text")
+        logger.error("Cannot read document")
+        return False
+    return True
+
+
+def display_file_read_error(e: Exception, file_name: str) -> NoReturn:
+    st.error("Error reading file. Make sure the file is not corrupted or encrypted")
+    logger.error(f"{e.__class__.__name__}: {e}. Extension: {file_name.split('.')[-1]}")
+    st.stop()
+
+
+@st.cache_data(show_spinner=False)
+def is_open_ai_key_valid(openai_api_key, model: str) -> bool:
+    if model == "debug":
+        return True
+
+    if not openai_api_key:
+        st.error("Please enter your OpenAI API key in the sidebar!")
+        return False
+    try:
+        openai.ChatCompletion.create(
+            model=model,
+            messages=[{"role": "user", "content": "test"}],
+            api_key=openai_api_key,
+        )
+    except Exception as e:
+        st.error(f"{e.__class__.__name__}: {e}")
+        logger.error(f"{e.__class__.__name__}: {e}")
+        return False
+
+    return True