Update app.py

app.py

@@ -15,7 +15,9 @@ load_dotenv()
 HUGGINGFACE_TOKEN = os.getenv("HUGGINGFACE_TOKEN")
 
 global_data = {'models': {}, 'tokens': {k: k + '_token' for k in ['eos', 'pad', 'padding', 'unk', 'bos', 'sep', 'cls', 'mask']}}
+
 model_configs = [{"repo_id": "Hjgugugjhuhjggg/mergekit-ties-tzamfyy-Q2_K-GGUF", "filename": "mergekit-ties-tzamfyy-q2_k.gguf", "name": "my_model"}]
+
 models = {}
 
 def load_model(model_config):
@@ -48,7 +50,7 @@ def generate_model_response(model, inputs):
     try:
         if model is None:
             return "Model loading failed."
-        response = model(inputs, max_tokens=1000) #Reduced chunk size for safety
+        response = model(inputs, max_tokens=512) #max_tokens adjusted for practicality
         return remove_duplicates(response['choices'][0]['text'])
     except Exception as e:
         print(f"Error generating response: {e}")
@@ -63,15 +65,14 @@ app.add_middleware(
 @app.post("/generate")
 async def generate(request: ChatRequest):
     inputs = normalize_input(request.message)
-    chunk_size = 400 # Reduced chunk size further for this model
+    chunk_size = 400  #Reduced chunk size
     chunks = [inputs[i:i + chunk_size] for i in range(0, len(inputs), chunk_size)]
     overall_response = ""
-
     for chunk in chunks:
         with ThreadPoolExecutor() as executor:
             futures = [executor.submit(generate_model_response, model, chunk) for model in models.values()]
             responses = [{'model': name, 'response': future.result()} for name, future in zip(models, as_completed(futures))]
-        for response in responses:  #Simplified response processing
+        for response in responses:
             overall_response += f"**{response['model']}:**\n{response['response']}\n\n"
     return {"response": overall_response}