Update app.py

app.py

@@ -37,16 +37,7 @@ model_name = "ggml-model-q4_1.gguf"
 
 #snapshot_download(repo_id=repo_name, local_dir=".", allow_patterns=model_name)
 
-model = Llama(
-    model_path=model_name,
-    n_ctx=2000,
-    n_parts=1,
-    #n_batch=100,
-    logits_all=True,
-    #n_threads=12,
-    verbose=True,
-    n_gqa=8       #must be set for 70b models
-)
+
 
 
 def get_message_tokens(model, role, content):
@@ -84,6 +75,7 @@ def generate_tokens(model, generator):
     for token in generator:            
         if token == model.token_eos() or stop_generation:
             stop_generation = False
+            app.logger.info('Abort generating')
             yield b''  # End of chunk
             break
             
@@ -111,6 +103,9 @@ def generate_unknown_response():
 def generate_search_request():
     global stop_generation
     stop_generation = False
+
+    
+    
     data = request.get_json()
     app.logger.info(data)
     user_query = data.get("query", "")
@@ -126,6 +121,17 @@ def generate_search_request():
     top_k = 20
     return_full_text = parameters.get("return_full_text", False)
 
+    model = Llama(
+        model_path=model_name,
+        n_ctx=2000,
+        n_parts=1,
+        #n_batch=100,
+        logits_all=True,
+        #n_threads=12,
+        verbose=True,
+        n_gqa=8       #must be set for 70b models
+    )
+    
     tokens = get_system_tokens_for_preprompt(model, preprompt)
     tokens.append(LINEBREAK_TOKEN)        
     
@@ -146,6 +152,7 @@ def generate_search_request():
 def generate_response():
     global stop_generation
     stop_generation = False
+
     
     data = request.get_json()
     app.logger.info(data)
@@ -163,6 +170,16 @@ def generate_response():
     return_full_text = parameters.get("return_full_text", False)
 
     
+    model = Llama(
+        model_path=model_name,
+        n_ctx=2000,
+        n_parts=1,
+        #n_batch=100,
+        logits_all=True,
+        #n_threads=12,
+        verbose=True,
+        n_gqa=8       #must be set for 70b models
+    )
     
     # Generate the response
     #system_tokens = get_system_tokens(model)