Daniel Marques commited on
Commit
e845546
1 Parent(s): b775dc2

feat: add stream

Browse files
Files changed (3) hide show
  1. main.py +3 -3
  2. run_localGPT.py +3 -2
  3. static/index.html +3 -3
main.py CHANGED
@@ -44,8 +44,8 @@ DB = Chroma(
44
 
45
  RETRIEVER = DB.as_retriever()
46
 
47
- LLM = load_model(device_type=DEVICE_TYPE, model_id=MODEL_ID, model_basename=MODEL_BASENAME, stream=False)
48
- prompt, memory = get_prompt_template(promptTemplate_type="llama", history=False)
49
 
50
  template = """you are a helpful, respectful and honest assistant.
51
  Your name is Katara llma. You should only use the source documents provided to answer the questions.
@@ -118,7 +118,7 @@ def run_ingest_route():
118
  )
119
 
120
  RETRIEVER = DB.as_retriever()
121
- prompt, memory = get_prompt_template(promptTemplate_type="llama", history=False)
122
 
123
  QA = RetrievalQA.from_chain_type(
124
  llm=LLM,
 
44
 
45
  RETRIEVER = DB.as_retriever()
46
 
47
+ LLM = load_model(device_type=DEVICE_TYPE, model_id=MODEL_ID, model_basename=MODEL_BASENAME)
48
+ prompt, memory = get_prompt_template(promptTemplate_type="llama", history=True)
49
 
50
  template = """you are a helpful, respectful and honest assistant.
51
  Your name is Katara llma. You should only use the source documents provided to answer the questions.
 
118
  )
119
 
120
  RETRIEVER = DB.as_retriever()
121
+ prompt, memory = get_prompt_template(promptTemplate_type="llama", history=True)
122
 
123
  QA = RetrievalQA.from_chain_type(
124
  llm=LLM,
run_localGPT.py CHANGED
@@ -38,7 +38,7 @@ from constants import (
38
 
39
 
40
 
41
- def load_model(device_type, model_id, model_basename=None, LOGGING=logging, stream=False):
42
  """
43
  Select a model for text generation using the HuggingFace library.
44
  If you are running this for the first time, it will download a model for you.
@@ -56,7 +56,6 @@ def load_model(device_type, model_id, model_basename=None, LOGGING=logging, stre
56
  Raises:
57
  ValueError: If an unsupported model or device type is provided.
58
  """
59
- streamer = TextStreamer(tokenizer, skip_prompt=stream)
60
 
61
  logging.info(f"Loading Model: {model_id}, on: {device_type}")
62
  logging.info("This action can take a few minutes!")
@@ -80,6 +79,8 @@ def load_model(device_type, model_id, model_basename=None, LOGGING=logging, stre
80
 
81
  # Create a pipeline for text generation
82
 
 
 
83
  pipe = pipeline(
84
  "text-generation",
85
  model=model,
 
38
 
39
 
40
 
41
+ def load_model(device_type, model_id, model_basename=None, LOGGING=logging):
42
  """
43
  Select a model for text generation using the HuggingFace library.
44
  If you are running this for the first time, it will download a model for you.
 
56
  Raises:
57
  ValueError: If an unsupported model or device type is provided.
58
  """
 
59
 
60
  logging.info(f"Loading Model: {model_id}, on: {device_type}")
61
  logging.info("This action can take a few minutes!")
 
79
 
80
  # Create a pipeline for text generation
81
 
82
+ streamer = TextStreamer(tokenizer, skip_prompt=True)
83
+
84
  pipe = pipeline(
85
  "text-generation",
86
  model=model,
static/index.html CHANGED
@@ -6,7 +6,7 @@
6
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
7
  <!-- Bootstrap CSS -->
8
 
9
- <link rel="icon" type="image" href="static\social_icons\favicon.png" />
10
  <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
11
  <link
12
  href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css"
@@ -20,7 +20,7 @@
20
  crossorigin="anonymous"
21
  ></script>
22
 
23
- <title>localGPT</title>
24
  </head>
25
  <style>
26
  @import url("https://fonts.googleapis.com/css2?family=Play:wght@400;700&display=swap");
@@ -56,7 +56,7 @@
56
  }
57
  </script>
58
  <div class="container">
59
- <div class="titlegpt">KATARA LLMA</div>
60
  <button
61
  type="button"
62
  style="
 
6
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
7
  <!-- Bootstrap CSS -->
8
 
9
+ <!-- <link rel="icon" type="image" href="static\social_icons\favicon.png" /> -->
10
  <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
11
  <link
12
  href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css"
 
20
  crossorigin="anonymous"
21
  ></script>
22
 
23
+ <title>Katara llama</title>
24
  </head>
25
  <style>
26
  @import url("https://fonts.googleapis.com/css2?family=Play:wght@400;700&display=swap");
 
56
  }
57
  </script>
58
  <div class="container">
59
+ <div class="titlegpt">Katara llama</div>
60
  <button
61
  type="button"
62
  style="