Update app.py

app.py

@@ -1,44 +1,46 @@
 import gradio as gr
+import torch
+import transformers
+from transformers import AutoTokenizer
+from langchain import LLMChain, HuggingFacePipeline, PromptTemplate
 
 def greet(text):
- 
- from langchain.chat_models import ChatOllama
- from langchain.document_loaders import WebBaseLoader
- from langchain.chains.summarize import load_summarize_chain
- 
- loader = WebBaseLoader("https://lilianweng.github.io/posts/2023-06-23-agent/")
- docs = loader.load()
- 
- llm = ChatOllama(temperature=0, model_name="falcon:7b")
- chain = load_summarize_chain(llm, chain_type="stuff")
- 
- chain.run(docs)
 
- from langchain.chains.llm import LLMChain
- from langchain.prompts import PromptTemplate
- from langchain.chains.combine_documents.stuff import StuffDocumentsChain
- 
- # Define prompt
- prompt_template = """Write a concise summary of the following:
- "{text}"
- CONCISE SUMMARY:"""
- prompt = PromptTemplate.from_template(prompt_template)
- 
- # Define LLM chain
- llm = ChatOllama(temperature=0, model_name="falcon:7b")
- llm_chain = LLMChain(llm=llm, prompt=prompt)
- 
- # Define StuffDocumentsChain
- stuff_chain = StuffDocumentsChain(
- llm_chain=llm_chain, document_variable_name="text"
+ model = "meta-llama/Llama-2-7b-chat-hf"
+ tokenizer = AutoTokenizer.from_pretrained(model)
+ 
+ pipeline = transformers.pipeline(
+ "text-generation",
+ model=model,
+ tokenizer=tokenizer,
+ torch_dtype=torch.bfloat16,
+ trust_remote_code=True,
+ device_map="auto",
+ max_length=1000,
+ do_sample=True,
+ top_k=10,
+ num_return_sequences=1,
+ eos_token_id=tokenizer.eos_token_id
  )
- 
- docs = loader.load()
- summary = stuff_chain.run(docs)
+
+ llm = HuggingFacePipeline(pipeline = pipeline, model_kwargs = {'temperature':0})
+
+ template = """
+ Write a summary of the following text delimited by triple backticks.
+ Return your response which covers the key points of the text.
+ ```{text}```
+ SUMMARY:
+ """
+ 
+ prompt = PromptTemplate(template=template, input_variables=["text"])
+ llm_chain = LLMChain(prompt=prompt, llm=llm)
+
+ summary = llm_chain.run(text)
  
  return summary
 
 with gr.Blocks() as demo:
+ 
  text = gr.Textbox(label="Text")
  summary = gr.Textbox(label="Summary")
  greet_btn = gr.Button("Submit")