Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -2,11 +2,10 @@ from transformers import T5Tokenizer, T5ForConditionalGeneration
|
|
2 |
from langchain.llms import HuggingFacePipeline
|
3 |
from langchain.prompts import PromptTemplate
|
4 |
from langchain.chains import RetrievalQA
|
5 |
-
from
|
6 |
-
from
|
7 |
-
from
|
8 |
from langchain.text_splitter import CharacterTextSplitter
|
9 |
-
from langchain_community.document_loaders import WikipediaLoader
|
10 |
from transformers import pipeline
|
11 |
|
12 |
# Load T5-small model and tokenizer
|
@@ -26,19 +25,16 @@ text_generation_pipeline = pipeline(
|
|
26 |
# Create a LangChain LLM from the pipeline
|
27 |
llm = HuggingFacePipeline(pipeline=text_generation_pipeline)
|
28 |
|
29 |
-
# Load and process documents
|
30 |
-
|
31 |
-
|
32 |
-
|
33 |
-
# Load content from Wikipedia
|
34 |
-
loader = WikipediaLoader(query="Artificial neuron", load_max_docs=1)
|
35 |
documents = loader.load()
|
36 |
-
|
37 |
text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
|
38 |
texts = text_splitter.split_documents(documents)
|
39 |
|
40 |
-
# Create embeddings
|
41 |
-
embeddings = HuggingFaceEmbeddings()
|
|
|
|
|
42 |
db = Chroma.from_documents(texts, embeddings)
|
43 |
|
44 |
# Create a retriever
|
|
|
2 |
from langchain.llms import HuggingFacePipeline
|
3 |
from langchain.prompts import PromptTemplate
|
4 |
from langchain.chains import RetrievalQA
|
5 |
+
from langchain.embeddings import HuggingFaceEmbeddings
|
6 |
+
from langchain.vectorstores import Chroma
|
7 |
+
from langchain.document_loaders import TextLoader
|
8 |
from langchain.text_splitter import CharacterTextSplitter
|
|
|
9 |
from transformers import pipeline
|
10 |
|
11 |
# Load T5-small model and tokenizer
|
|
|
25 |
# Create a LangChain LLM from the pipeline
|
26 |
llm = HuggingFacePipeline(pipeline=text_generation_pipeline)
|
27 |
|
28 |
+
# Load and process documents from a local file
|
29 |
+
loader = TextLoader("NeuralNetworkWikipedia.txt")
|
|
|
|
|
|
|
|
|
30 |
documents = loader.load()
|
|
|
31 |
text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
|
32 |
texts = text_splitter.split_documents(documents)
|
33 |
|
34 |
+
# Create embeddings using a smaller model
|
35 |
+
embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
|
36 |
+
|
37 |
+
# Create vector store
|
38 |
db = Chroma.from_documents(texts, embeddings)
|
39 |
|
40 |
# Create a retriever
|