import os import gc import psutil import cachetools from pydantic import BaseModel from llama_cpp import Llama from concurrent.futures import ThreadPoolExecutor, as_completed import re import httpx import asyncio import gradio as gr import torch from dotenv import load_dotenv from fastapi import FastAPI, Request from fastapi.responses import JSONResponse import uvicorn from threading import Thread import gptcache from sklearn.metrics.pairwise import cosine_similarity from sklearn.feature_extraction.text import TfidfVectorizer import nltk from nltk.corpus import stopwords from huggingface_hub import download_url nltk.download('stopwords') load_dotenv() HUGGINGFACE_TOKEN = os.getenv("HUGGINGFACE_TOKEN") cache = cachetools.TTLCache(maxsize=100, ttl=60) global_data = { 'models': {}, 'tokensxx': { 'eos': '<|end_of-text|>', 'pad': '', 'padding': '', 'unk': '', 'bos': '<|begin_of_text|>', 'sep': '<|sep|>', 'cls': '<|cls|>', 'mask': '', 'eot': '<|eot_id|>', 'eom': '<|eom_id|>', 'lf': '<|0x0A|>' }, 'tokens': { 'eos': 'eos_token', 'pad': 'pad_token', 'padding': 'padding_token', 'unk': 'unk_token', 'bos': 'bos_token', 'sep': 'sep_token', 'cls': 'cls_token', 'mask': 'mask_token' }, 'model_metadata': {}, 'eos': {}, 'pad': {}, 'padding': {}, 'unk': {}, 'bos': {}, 'sep': {}, 'cls': {}, 'mask': {}, 'eot': {}, 'eom': {}, 'lf': {}, 'max_tokens': {}, 'tokenizers': {}, 'model_params': {}, 'model_size': {}, 'model_ftype': {}, 'n_ctx_train': {}, 'n_embd': {}, 'n_layer': {}, 'n_head': {}, 'n_head_kv': {}, 'n_rot': {}, 'n_swa': {}, 'n_embd_head_k': {}, 'n_embd_head_v': {}, 'n_gqa': {}, 'n_embd_k_gqa': {}, 'n_embd_v_gqa': {}, 'f_norm_eps': {}, 'f_norm_rms_eps': {}, 'f_clamp_kqv': {}, 'f_max_alibi_bias': {}, 'f_logit_scale': {}, 'n_ff': {}, 'n_expert': {}, 'n_expert_used': {}, 'causal_attn': {}, 'pooling_type': {}, 'rope_type': {}, 'rope_scaling': {}, 'freq_base_train': {}, 'freq_scale_train': {}, 'n_ctx_orig_yarn': {}, 'rope_finetuned': {}, 'ssm_d_conv': {}, 'ssm_d_inner': {}, 'ssm_d_state': {}, 'ssm_dt_rank': {}, 'ssm_dt_b_c_rms': {}, 'vocab_type': {}, 'model_type': {}, "general.architecture": {}, "general.type": {}, "general.name": {}, "general.finetune": {}, "general.basename": {}, "general.size_label": {}, "general.license": {}, "general.license.link": {}, "general.tags": {}, "general.languages": {}, "general.organization": {}, "general.base_model.count": {}, 'general.file_type': {}, "phi3.context_length": {}, "phi3.rope.scaling.original_context_length": {}, "phi3.embedding_length": {}, "phi3.feed_forward_length": {}, "phi3.block_count": {}, "phi3.attention.head_count": {}, "phi3.attention.head_count_kv": {}, "phi3.attention.layer_norm_rms_epsilon": {}, "phi3.rope.dimension_count": {}, "phi3.rope.freq_base": {}, "phi3.attention.sliding_window": {}, "phi3.rope.scaling.attn_factor": {}, "llama.block_count": {}, "llama.context_length": {}, "llama.embedding_length": {}, "llama.feed_forward_length": {}, "llama.attention.head_count": {}, "llama.attention.head_count_kv": {}, "llama.rope.freq_base": {}, "llama.attention.layer_norm_rms_epsilon": {}, "llama.attention.key_length": {}, "llama.attention.value_length": {}, "llama.vocab_size": {}, "llama.rope.dimension_count": {}, "deepseek2.block_count": {}, "deepseek2.context_length": {}, "deepseek2.embedding_length": {}, "deepseek2.feed_forward_length": {}, "deepseek2.attention.head_count": {}, "deepseek2.attention.head_count_kv": {}, "deepseek2.rope.freq_base": {}, "deepseek2.attention.layer_norm_rms_epsilon": {}, "deepseek2.expert_used_count": {}, "deepseek2.leading_dense_block_count": {}, "deepseek2.vocab_size": {}, "deepseek2.attention.kv_lora_rank": {}, "deepseek2.attention.key_length": {}, "deepseek2.attention.value_length": {}, "deepseek2.expert_feed_forward_length": {}, "deepseek2.expert_count": {}, "deepseek2.expert_shared_count": {}, "deepseek2.expert_weights_scale": {}, "deepseek2.rope.dimension_count": {}, "deepseek2.rope.scaling.type": {}, "deepseek2.rope.scaling.factor": {}, "deepseek2.rope.scaling.yarn_log_multiplier": {}, "qwen2.block_count": {}, "qwen2.context_length": {}, "qwen2.embedding_length": {}, "qwen2.feed_forward_length": {}, "qwen2.attention.head_count": {}, "qwen2.attention.head_count_kv": {}, "qwen2.rope.freq_base": {}, "qwen2.attention.layer_norm_rms_epsilon": {}, "general.version": {}, "general.datasets": {}, "tokenizer.ggml.model": {}, "tokenizer.ggml.pre": {}, "tokenizer.ggml.tokens": {}, "tokenizer.ggml.token_type": {}, "tokenizer.ggml.merges": {}, "tokenizer.ggml.bos_token_id": {}, "tokenizer.ggml.eos_token_id": {}, "tokenizer.ggml.unknown_token_id": {}, "tokenizer.ggml.padding_token_id": {}, "tokenizer.ggml.add_bos_token": {}, "tokenizer.ggml.add_eos_token": {}, "tokenizer.ggml.add_space_prefix": {}, "tokenizer.chat_template": {}, "quantize.imatrix.file": {}, "quantize.imatrix.dataset": {}, "quantize.imatrix.entries_count": {}, "quantize.imatrix.chunks_count": {}, "general.quantization_version": {}, 'n_lora_q': {}, 'n_lora_kv': {}, 'n_expert_shared': {}, 'n_ff_exp': {}, "n_layer_dense_lead": {}, "expert_weights_scale": {}, "rope_yarn_log_mul": {}, 'model_type': {}, 'eval': {}, 'time': {}, 'token': {}, 'tokens': {}, 'pads': {}, 'model': {}, 'base': {}, 'model_base': {}, 'perhaps': {}, 'word': {}, 'words': {}, 'start': {}, 'stop': {}, 'run': {}, 'runs': {}, 'ms': {}, 'vocabulary': {}, 'timeout': {}, 'load': {}, 'load_time': {}, 'bas': {}, 'tok': {}, 'second': {}, 'seconds': {}, 'graph': {}, 'load_model': {}, 'end': {}, 'llama_perf_context_print': {}, 'llm_load_print_meta': {} } model_configs = [ {"repo_id": "Hjgugugjhuhjggg/testing_semifinal-Q2_K-GGUF", "filename": "testing_semifinal-q2_k.gguf", "name": "testing"}, {"repo_id": "bartowski/Llama-3.2-3B-Instruct-uncensored-GGUF", "filename": "Llama-3.2-3B-Instruct-uncensored-Q2_K.gguf", "name": "Llama-3.2-3B-Instruct"}, {"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-70B", "filename": "meta-llama-3.1-70B", "name": "Meta-Llama-3.1-70B"}, {"repo_id": "Hhhbvvkgh/Heidi-Llama-v4-Q2_K-GGUF", "filename": "heidi-llama-v4-q2_k.gguf", "name": "Heidi-Llama-V4"} ] def normalize_input(input_text): stop_words = set(stopwords.words('english')) words = input_text.split() filtered_words = [word for word in words if word.lower() not in stop_words] return " ".join(filtered_words) async def load_models(): for model in model_configs: model_path = os.path.join("models", model["filename"]) if not os.path.exists(model_path): url = f"https://huggingface.co/{model['repo_id']}/resolve/main/{model['filename']}" download_url(url, model_path) global_data['models'][model['name']] = Llama(model_path) async def generate_model_response(model, inputs): try: response = await model.generate(inputs) return response except Exception as e: return {"error": str(e)} def get_best_response(responses): if not responses: return {"error": "No valid responses from models."} return max(responses, key=lambda x: x['score']) async def process_message(message): inputs = normalize_input(message) tasks = [generate_model_response(model, inputs) for model in global_data['models'].values()] responses = await asyncio.gather(*tasks) best_response = get_best_response(responses) return best_response app = FastAPI() @app.post("/generate") async def generate(request: Request): try: body = await request.json() response = await process_message(body['message']) return JSONResponse(content={"response": response}) except Exception as e: return JSONResponse(content={"error": str(e)}) def run_uvicorn(): uvicorn.run(app, host="0.0.0.0", port=7860) iface = gr.Interface( fn=process_message, inputs=gr.Textbox(lines=2, placeholder="Enter your message here..."), outputs=gr.Markdown(), title="Multi-Model LLM API (CPU Optimized)", description="" ) def run_gradio(): iface.launch(server_port=7862, prevent_thread_lock=True) if __name__ == "__main__": asyncio.run(load_models()) Thread(target=run_uvicorn).start() Thread(target=run_gradio).start() asyncio.get_event_loop().run_forever()