add cache_resource

.gitignore

@@ -0,0 +1 @@
+.env

app.py

@@ -13,62 +13,67 @@ load_dotenv('.env')
 
 st.header("MKOM UGM RAG App")
 
-# Only keep post title, headers, and content from the full HTML.
-bs4_strainer = bs4.SoupStrainer(class_=("post-title", "post-header", "post-content"))    
-loader = WebBaseLoader(
-    web_paths=(
-        "https://um.ugm.ac.id/ragam-seleksi-pascasarjana/",
-        "https://um.ugm.ac.id/persyaratan-pendaftaran-magister/",
-        "https://um.ugm.ac.id/persyaratan-pendaftaran-program-spesialis/",
-        "https://um.ugm.ac.id/persyaratan-pendaftaran-subspesialis/",
-        "https://um.ugm.ac.id/persyaratan-pendaftaran-doktor/",
-        "https://um.ugm.ac.id/prosedur-pendaftaran-magister/",
-        "https://um.ugm.ac.id/prosedur-pendaftaran-program-spesialis/",
-        "https://um.ugm.ac.id/prosedur-pendaftaran-program-subspesialis/",
-        "https://um.ugm.ac.id/prosedur-pendaftaran-doktor-2/",
-        "https://um.ugm.ac.id/program-studi-program-magister-2/",
-        "https://um.ugm.ac.id/program-studi-dan-daya-tampung-program-spesialis/",
-        "https://um.ugm.ac.id/program-studi-program-doktor/",
-        "https://um.ugm.ac.id/jadwal-seleksi-magister-dan-doktor/",
-        "https://um.ugm.ac.id/jadwal-kegiatan-seleksi-program-spesialis/",
-        "https://mkom.ugm.ac.id/alur-pendaftaran-magister/",
-        "https://mkom.ugm.ac.id/informasi-pendaftaran-program-pra-s2-ilmu-komputer/",
-        "https://mkom.ugm.ac.id/informasi-pendaftaran-program-s2-magister/",
-        "https://mkom.ugm.ac.id/program-dual-degree-double-degree-magister-ilmu-komputer/",
-        "https://mkom.ugm.ac.id/informasi-pendaftaran-program-s3-doktor/"
-    ),
-    bs_kwargs={"parse_only": bs4_strainer},
-)
-docs = loader.load()
-text_splitter = RecursiveCharacterTextSplitter(
-    chunk_size=1000, chunk_overlap=200, add_start_index=True
-)
-all_splits = text_splitter.split_documents(docs)
-vectorstore = FAISS.from_documents(all_splits, HuggingFaceEmbeddings(model_name='firqaaa/indo-sentence-bert-base'))
-retriever = vectorstore.as_retriever(search_type="similarity", search_kwargs={"k": 6})
-llm = ChatCohere(model="command-r")
+@st.cache_resource
+def get_rag_chain():
+    # Only keep post title, headers, and content from the full HTML.
+    bs4_strainer = bs4.SoupStrainer(class_=("post-title", "post-header", "post-content"))    
+    loader = WebBaseLoader(
+        web_paths=(
+            "https://um.ugm.ac.id/ragam-seleksi-pascasarjana/",
+            "https://um.ugm.ac.id/persyaratan-pendaftaran-magister/",
+            "https://um.ugm.ac.id/persyaratan-pendaftaran-program-spesialis/",
+            "https://um.ugm.ac.id/persyaratan-pendaftaran-subspesialis/",
+            "https://um.ugm.ac.id/persyaratan-pendaftaran-doktor/",
+            "https://um.ugm.ac.id/prosedur-pendaftaran-magister/",
+            "https://um.ugm.ac.id/prosedur-pendaftaran-program-spesialis/",
+            "https://um.ugm.ac.id/prosedur-pendaftaran-program-subspesialis/",
+            "https://um.ugm.ac.id/prosedur-pendaftaran-doktor-2/",
+            "https://um.ugm.ac.id/program-studi-program-magister-2/",
+            "https://um.ugm.ac.id/program-studi-dan-daya-tampung-program-spesialis/",
+            "https://um.ugm.ac.id/program-studi-program-doktor/",
+            "https://um.ugm.ac.id/jadwal-seleksi-magister-dan-doktor/",
+            "https://um.ugm.ac.id/jadwal-kegiatan-seleksi-program-spesialis/",
+            "https://mkom.ugm.ac.id/alur-pendaftaran-magister/",
+            "https://mkom.ugm.ac.id/informasi-pendaftaran-program-pra-s2-ilmu-komputer/",
+            "https://mkom.ugm.ac.id/informasi-pendaftaran-program-s2-magister/",
+            "https://mkom.ugm.ac.id/program-dual-degree-double-degree-magister-ilmu-komputer/",
+            "https://mkom.ugm.ac.id/informasi-pendaftaran-program-s3-doktor/"
+        ),
+        bs_kwargs={"parse_only": bs4_strainer},
+    )
+    docs = loader.load()
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1000, chunk_overlap=200, add_start_index=True
+    )
+    all_splits = text_splitter.split_documents(docs)
+    vectorstore = FAISS.from_documents(all_splits, HuggingFaceEmbeddings(model_name='firqaaa/indo-sentence-bert-base'))
+    retriever = vectorstore.as_retriever(search_type="similarity", search_kwargs={"k": 6})
+    llm = ChatCohere(model="command-r")
 
-def format_docs(docs):
-    return "\n\n".join(doc.page_content for doc in docs)
+    def format_docs(docs):
+        return "\n\n".join(doc.page_content for doc in docs)
 
-template = """Gunakan konteks berikut untuk menjawab pertanyaan pada bagian akhir.
-Jika kamu tidak tahu jawabannya, katakan saja bahwa kamu tidak tahu, jangan mencoba untuk mengarang jawaban.
-Selalu katakan "Terima kasih sudah bertanya!" pada setiap akhir jawaban.
+    template = """Gunakan konteks berikut untuk menjawab pertanyaan pada bagian akhir.
+    Jika kamu tidak tahu jawabannya, katakan saja bahwa kamu tidak tahu, jangan mencoba untuk mengarang jawaban.
+    Selalu katakan "Terima kasih sudah bertanya!" pada setiap akhir jawaban.
 
-{context}
+    {context}
 
-Pertanyaan: {question}
+    Pertanyaan: {question}
 
-Jawaban:"""
+    Jawaban:"""
 
-custom_rag_prompt = PromptTemplate.from_template(template)
+    custom_rag_prompt = PromptTemplate.from_template(template)
 
-rag_chain = (
-    {"context": retriever | format_docs, "question": RunnablePassthrough()}
-    | custom_rag_prompt
-    | llm
-    | StrOutputParser()
-)
+    rag_chain = (
+        {"context": retriever | format_docs, "question": RunnablePassthrough()}
+        | custom_rag_prompt
+        | llm
+        | StrOutputParser()
+    )
+    return rag_chain
+
+rag_chain = get_rag_chain()
 
 question = st.text_input("Tanya ujian masuk Pascasarjana Universitas Gadjah Mada")
 if question:

rag_notebook.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 94,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -11,7 +11,7 @@
        "2879"
       ]
      },
-     "execution_count": 94,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -53,7 +53,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 95,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -81,7 +81,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 96,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -90,7 +90,7 @@
        "111"
       ]
      },
-     "execution_count": 96,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -108,7 +108,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 97,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
@@ -117,7 +117,7 @@
        "835"
       ]
      },
-     "execution_count": 97,
+     "execution_count": 15,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -128,7 +128,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 98,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -138,7 +138,7 @@
        " 'start_index': 4748}"
       ]
      },
-     "execution_count": 98,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -149,7 +149,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 99,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [
     {
@@ -170,7 +170,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [
     {
@@ -179,7 +179,7 @@
        "6"
       ]
      },
-     "execution_count": 73,
+     "execution_count": 18,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -194,7 +194,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [
     {
@@ -224,7 +224,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -239,7 +239,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [
     {
@@ -248,7 +248,7 @@
        "[HumanMessage(content=\"You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know. Use three sentences maximum and keep the answer concise.\\nQuestion: filler question \\nContext: filler context \\nAnswer:\")]"
       ]
      },
-     "execution_count": 76,
+     "execution_count": 21,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -267,7 +267,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 22,
    "metadata": {},
    "outputs": [
     {
@@ -287,14 +287,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 23,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Penerimaan Mahasiswa Baru (PMB) Program Pascasarjana jalur reguler terdiri dari dua skema: biaya sendiri dan kerja sama. Skema kerja sama diperuntukkan bagi penerima beasiswa atau yang pendidikannya dibiayai mitra UGM.Penerimaan Mahasiswa Baru (PMB) Program Pascasarjana jalur reguler terdiri dari dua skema: biaya sendiri dan kerja sama. Skema kerja sama diperuntukkan bagi penerima beasiswa atau yang pendidikannya dibiayai mitra UGM."
+      "Penerimaan Mahasiswa Baru (PMB) Program Pascasarjana jalur reguler dilakukan melalui tiga skema: biaya sendiri, kerja sama, dan pelamar beasiswa. Jalur ini terbuka untuk semua pendaftar tanpa batasan institusi atau negara asal ijazah.Penerimaan Mahasiswa Baru (PMB) Program Pascasarjana jalur reguler dilakukan melalui tiga skema: biaya sendiri, kerja sama, dan pelamar beasiswa. Jalur ini terbuka untuk semua pendaftar tanpa batasan institusi atau negara asal ijazah."
      ]
     }
    ],
@@ -320,16 +320,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 24,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "'Nomor telepon untuk PROGRAM MAGISTER (S2) SEMESTER GASAL TA 2024/2025 program studi Magister Akuntansi adalah 274-513109, 0274-548516, dan 085292000355.\\n\\nTerima kasih sudah bertanya!'"
+       "'274-513109, 0274-548516, 085292000355.\\n\\nTerima kasih sudah bertanya!'"
       ]
      },
-     "execution_count": 93,
+     "execution_count": 24,
      "metadata": {},
      "output_type": "execute_result"
     }