Upload 5 files (#16)

- Upload 5 files (359ea037ee72d51eb931689359a044c9eb22ca6f)

app.py

@@ -30,9 +30,15 @@ st.title("Instructor XL Embeddings")
 
 
 st.write(
- "The app compares the performance of the Instructor-XL Embedding Model on the text from AMD's Q1 2020 Earnings Call Transcript."
+ """The app compares the performance of different instructions using the Instructor-XL Embedding Model on the text from AMD's Q1 2020 Earnings Call Transcript. 
+ The app uses a two stage retreival process: 
+ 1. BM-25 to filter the results based on keyword matching, 
+ 2. Instructor-XL to perform Semantic Search."""
 )
 
+use_bm25 = st.checkbox('Use BM25 for filtering results')
+
+
 data = get_data()
 
 
@@ -86,6 +92,7 @@ with col1:
  st.number_input("Number of Results to query", 1, 15, value=5)
  )
 
+
 corpus, bm25 = get_bm25_model(data)
 
 tokenized_query = preprocess_text(query_text).split()
@@ -127,35 +134,36 @@ index_mapping = {
 }
 
 with col2:
- with st.form("my_form"):
- text_embedding_instruction = st.selectbox(
- "Select instruction for Text Embedding",
- text_embedding_instructions_choice,
- )
+ text_embedding_instruction = st.selectbox(
+ "Select instruction for Text Embedding",
+ text_embedding_instructions_choice,
+ )
+ pinecone_index_name = index_mapping[text_embedding_instruction]
+ pinecone.init(
+ api_key=st.secrets[f"pinecone_{pinecone_index_name}"],
+ environment="asia-southeast1-gcp-free",
+ )
 
- submitted = st.form_submit_button("Submit")
- if submitted:
- pinecone_index_name = index_mapping[text_embedding_instruction]
- pinecone.init(
- api_key=st.secrets[f"pinecone_{pinecone_index_name}"],
- environment="asia-southeast1-gcp-free",
- )
-
- pinecone_index = pinecone.Index(pinecone_index_name)
-
- matches = query_pinecone(
- dense_vec=dense_embedding_api, top_k=num_results, index=pinecone_index, indices=indices
- )
- context = format_query(matches)
- output_text = format_context(context)
-
- st.subheader("Retrieved Text:")
- for output in output_text:
- output = f"""{output}"""
- st.write(
- f"<ul><li><p>{output}</p></li></ul>",
- unsafe_allow_html=True,
- )
+ pinecone_index = pinecone.Index(pinecone_index_name)
+ 
+ if use_bm25==True:
+ matches = query_pinecone(
+ dense_vec=dense_embedding_api, top_k=num_results, index=pinecone_index, indices=indices
+ )
+ else:
+ matches = query_pinecone(
+ dense_vec=dense_embedding_api, top_k=num_results, index=pinecone_index, indices=None
+ )
+ context = format_query(matches)
+ output_text = format_context(context)
+
+ st.subheader("Retrieved Text:")
+ for output in output_text:
+ output = f"""{output}"""
+ st.write(
+ f"<ul><li><p>{output}</p></li></ul>",
+ unsafe_allow_html=True,
+ )
 
 
 file_text = retrieve_transcript()

utils/retriever.py

@@ -6,15 +6,23 @@ def query_pinecone(
  dense_vec,
  top_k,
  index,
- indices
+ indices=None
 ):
- xc = index.query(
- vector=dense_vec,
- top_k=top_k, 
- filter={"QA_Flag": {"$eq": "Answer"},
- "index": {"$in": indices}},
- include_metadata=True,
- )
+ if indices != None:
+ xc = index.query(
+ vector=dense_vec,
+ top_k=top_k, 
+ filter={"QA_Flag": {"$eq": "Answer"},
+ "index": {"$in": indices}},
+ include_metadata=True,
+ )
+ else:
+ xc = index.query(
+ vector=dense_vec,
+ top_k=top_k, 
+ filter={"QA_Flag": {"$eq": "Answer"}},
+ include_metadata=True,
+ )
  return xc["matches"]