ashutoshzade commited on
Commit
f69aa6c
·
verified ·
1 Parent(s): 43fe401

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +9 -13
app.py CHANGED
@@ -2,11 +2,10 @@ from transformers import T5Tokenizer, T5ForConditionalGeneration
2
  from langchain.llms import HuggingFacePipeline
3
  from langchain.prompts import PromptTemplate
4
  from langchain.chains import RetrievalQA
5
- from langchain_community.embeddings import HuggingFaceEmbeddings
6
- from langchain_community.vectorstores import Chroma
7
- from langchain_community.document_loaders import TextLoader
8
  from langchain.text_splitter import CharacterTextSplitter
9
- from langchain_community.document_loaders import WikipediaLoader
10
  from transformers import pipeline
11
 
12
  # Load T5-small model and tokenizer
@@ -26,19 +25,16 @@ text_generation_pipeline = pipeline(
26
  # Create a LangChain LLM from the pipeline
27
  llm = HuggingFacePipeline(pipeline=text_generation_pipeline)
28
 
29
- # Load and process documents
30
- #loader = TextLoader("https://en.wikipedia.org/wiki/Artificial_neuron")
31
-
32
-
33
- # Load content from Wikipedia
34
- loader = WikipediaLoader(query="Artificial neuron", load_max_docs=1)
35
  documents = loader.load()
36
-
37
  text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
38
  texts = text_splitter.split_documents(documents)
39
 
40
- # Create embeddings and vector store
41
- embeddings = HuggingFaceEmbeddings()
 
 
42
  db = Chroma.from_documents(texts, embeddings)
43
 
44
  # Create a retriever
 
2
  from langchain.llms import HuggingFacePipeline
3
  from langchain.prompts import PromptTemplate
4
  from langchain.chains import RetrievalQA
5
+ from langchain.embeddings import HuggingFaceEmbeddings
6
+ from langchain.vectorstores import Chroma
7
+ from langchain.document_loaders import TextLoader
8
  from langchain.text_splitter import CharacterTextSplitter
 
9
  from transformers import pipeline
10
 
11
  # Load T5-small model and tokenizer
 
25
  # Create a LangChain LLM from the pipeline
26
  llm = HuggingFacePipeline(pipeline=text_generation_pipeline)
27
 
28
+ # Load and process documents from a local file
29
+ loader = TextLoader("NeuralNetworkWikipedia.txt")
 
 
 
 
30
  documents = loader.load()
 
31
  text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
32
  texts = text_splitter.split_documents(documents)
33
 
34
+ # Create embeddings using a smaller model
35
+ embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
36
+
37
+ # Create vector store
38
  db = Chroma.from_documents(texts, embeddings)
39
 
40
  # Create a retriever