CaesarFrenchLLM first test

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,14 @@
+# read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.10
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY . .
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,10 @@
+---
+title: CaesarFrenchLLM
+emoji: 💻
+colorFrom: red
+colorTo: yellow
+sdk: docker
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

caesarfrenchllm copy.py

@@ -0,0 +1,65 @@
+import torch
+import os
+os.environ['TRANSFORMERS_CACHE'] = "T:/CaesarLLModel/.cache"
+from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig, TextStreamer
+from vigogne.preprocess import generate_inference_chat_prompt
+
+if __name__ == "__main__":
+    
+    base_model_name_or_path = "bofenghuang/vigogne-2-7b-chat"
+    tokenizer = AutoTokenizer.from_pretrained(base_model_name_or_path, padding_side="right", use_fast=False,)
+    tokenizer.pad_token = tokenizer.eos_token
+
+    model = AutoModelForCausalLM.from_pretrained(
+        base_model_name_or_path,
+        torch_dtype=torch.float32,
+        device_map="auto",
+        offload_folder="T:/CaesarLLModel/.cache/offload"
+
+        # load_in_8bit=True,
+        # trust_remote_code=True,
+        # low_cpu_mem_usage=True,
+    )
+
+    # lora_model_name_or_path = ""
+    # model = PeftModel.from_pretrained(model, lora_model_name_or_path)
+
+    model.eval()
+
+    #if torch.__version__ >= "2":
+    #    model = torch.compile(model)
+    streamer = TextStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
+    def infer( user_query,temperature=0.1,top_p=1.0,top_k=0,max_new_tokens=512,**kwargs,):
+        prompt = generate_inference_chat_prompt(user_query, tokenizer=tokenizer)
+        input_ids = tokenizer(prompt, return_tensors="pt")["input_ids"].to(model.device)
+        input_length = input_ids.shape[1]
+
+        generated_outputs = model.generate(
+            input_ids=input_ids,
+            generation_config=GenerationConfig(
+                temperature=temperature,
+                do_sample=temperature > 0.0,
+                top_p=top_p,
+                top_k=top_k,
+                max_new_tokens=max_new_tokens,
+                eos_token_id=tokenizer.eos_token_id,
+                pad_token_id=tokenizer.pad_token_id,
+                **kwargs,
+            ),
+            streamer=streamer,
+            return_dict_in_generate=True,
+        )
+        generated_tokens = generated_outputs.sequences[0, input_length:]
+        generated_text = tokenizer.decode(generated_tokens, skip_special_tokens=True)
+        return generated_text
+    def chat(**kwargs):
+        history = []
+        while True:
+            user_input = input(">> <|user|>: ")
+            print(">> <|assistant|>: ", end="")
+            model_response = infer([*history, [user_input, ""]], **kwargs)
+
+            history.append([user_input, model_response])
+            return history[-1][1]
+            # print(f">> <|assistant|>: {history[-1][1]}")
+    chat()

caesarfrenchllm.py

@@ -0,0 +1,64 @@
+import torch
+import os
+#os.environ['TRANSFORMERS_CACHE'] = "T:/CaesarLLModel/.cache"
+from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig, TextStreamer
+from vigogne.preprocess import generate_inference_chat_prompt
+class CaesarFrenchLLM:
+    def __init__(self) -> None:
+        self.history = []
+        base_model_name_or_path = "bofenghuang/vigogne-2-7b-chat"
+        self.tokenizer = AutoTokenizer.from_pretrained(base_model_name_or_path, padding_side="right", use_fast=False,)
+        self.tokenizer.pad_token = self.tokenizer.eos_token
+
+        self.model = AutoModelForCausalLM.from_pretrained(
+            base_model_name_or_path,
+            torch_dtype=torch.float32,
+            device_map="auto",
+
+
+            # load_in_8bit=True,
+            # trust_remote_code=True,
+            # low_cpu_mem_usage=True,
+        )
+
+        # lora_model_name_or_path = ""
+        # model = PeftModel.from_pretrained(model, lora_model_name_or_path)
+
+        self.model.eval()
+
+        if torch.__version__ >= "2":
+            self.model = torch.compile(self.model)
+        self.streamer = TextStreamer(self.tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
+    def infer(self,user_query,temperature=0.1,top_p=1.0,top_k=0,max_new_tokens=512,**kwargs,):
+        prompt = generate_inference_chat_prompt(user_query, tokenizer=self.tokenizer)
+        input_ids = self.tokenizer(prompt, return_tensors="pt")["input_ids"].to(self.model.device)
+        input_length = input_ids.shape[1]
+
+        generated_outputs = self.model.generate(
+            input_ids=input_ids,
+            generation_config=GenerationConfig(
+                temperature=temperature,
+                do_sample=temperature > 0.0,
+                top_p=top_p,
+                top_k=top_k,
+                max_new_tokens=max_new_tokens,
+                eos_token_id=self.tokenizer.eos_token_id,
+                pad_token_id=self.tokenizer.pad_token_id,
+                **kwargs,
+            ),
+            streamer=self.streamer,
+            return_dict_in_generate=True,
+        )
+        generated_tokens = generated_outputs.sequences[0, input_length:]
+        generated_text = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
+        return generated_text
+    
+    def chat(self,user_input,**kwargs):
+        print(f">> <|user|>: {user_input}")
+        print(">> <|assistant|>: ", end="")
+        model_response = self.infer([*self.history, [user_input, ""]], **kwargs)
+
+        self.history.append([user_input, model_response])
+        return self.history[-1][1]
+            # print(f">> <|assistant|>: {history[-1][1]}")
+    

main.py

@@ -0,0 +1,37 @@
+import asyncio
+
+from fastapi import FastAPI, UploadFile, WebSocket, WebSocketDisconnect
+from fastapi.responses import FileResponse
+from caesarfrenchllm import CaesarFrenchLLM
+import uvicorn
+app = FastAPI()
+caesarmodel = CaesarFrenchLLM()
+
+@app.get("/")
+def caesaraihome():
+    return "Welcome to CaesarAI's API's and CaesarAINL."
+
+@app.websocket("/caesarfrenchllmws")
+async def caesarfrenchllmws(websocket: WebSocket):
+    # listen for connections
+    await websocket.accept()
+
+    try:
+        while True:
+            contents = await websocket.receive_json()
+            textresp = caesarmodel.chat(contents["message"])
+            final_response = {"message":textresp}
+            await websocket.send_json(final_response) # sends the buffer as bytes
+
+
+    except WebSocketDisconnect:
+        print("Client disconnected")
+
+
+async def main():
+    config = uvicorn.Config("main:app", port=7860, log_level="info",host="0.0.0.0",reload=True)
+    server = uvicorn.Server(config)
+    await server.serve()
+
+if __name__ == "__main__":
+    asyncio.run(main())

requirements.txt

@@ -0,0 +1,106 @@
+absl-py==1.4.0
+accelerate @ git+https://github.com/huggingface/accelerate.git@7befe580c2393b0a3d75a8aa75af79b8e57f84a4
+aiofiles==23.2.1
+aiohttp==3.8.5
+aiosignal==1.3.1
+altair==5.1.1
+annotated-types==0.5.0
+anyio==4.0.0
+appdirs==1.4.4
+async-timeout==4.0.3
+attrs==23.1.0
+bitsandbytes==0.41.1
+certifi==2023.7.22
+charset-normalizer==3.2.0
+click==8.1.7
+colorama==0.4.6
+contourpy==1.1.0
+cycler==0.11.0
+datasets==2.14.4
+dill==0.3.7
+docker-pycreds==0.4.0
+einops==0.6.1
+exceptiongroup==1.1.3
+fastapi==0.103.0
+ffmpy==0.3.1
+filelock==3.12.3
+fire==0.5.0
+fonttools==4.42.1
+frozenlist==1.4.0
+fsspec==2023.6.0
+gitdb==4.0.10
+GitPython==3.1.32
+gradio==3.41.2
+gradio_client==0.5.0
+h11==0.14.0
+httpcore==0.17.3
+httpx==0.24.1
+huggingface-hub==0.16.4
+idna==3.4
+importlib-resources==6.0.1
+Jinja2==3.1.2
+joblib==1.3.2
+jsonschema==4.19.0
+jsonschema-specifications==2023.7.1
+kiwisolver==1.4.5
+loralib==0.1.2
+MarkupSafe==2.1.3
+matplotlib==3.7.2
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.15
+networkx==3.1
+nltk==3.8.1
+numpy==1.25.2
+openai==0.27.10
+orjson==3.9.5
+packaging==23.1
+pandas==2.1.0
+pathtools==0.1.2
+peft @ git+https://github.com/huggingface/peft.git@0c9354bda98eb7f5348699e23ab752e8dca1e60e
+Pillow==10.0.0
+protobuf==4.24.2
+psutil==5.9.5
+pyarrow==13.0.0
+pydantic==2.3.0
+pydantic_core==2.6.3
+pydub==0.25.1
+pyparsing==3.0.9
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+PyYAML==6.0.1
+referencing==0.30.2
+regex==2023.8.8
+requests==2.31.0
+rouge-score==0.1.2
+rpds-py==0.10.0
+safetensors==0.3.3
+scipy==1.11.2
+semantic-version==2.10.0
+sentencepiece==0.1.99
+sentry-sdk==1.30.0
+setproctitle==1.3.2
+six==1.16.0
+smmap==5.0.0
+sniffio==1.3.0
+starlette==0.27.0
+sympy==1.12
+tenacity==8.2.3
+tensorboardX==2.6.2.2
+termcolor==2.3.0
+tiktoken==0.4.0
+tokenizers==0.13.3
+toolz==0.12.0
+torch==2.0.1
+tqdm==4.66.1
+transformers @ git+https://github.com/huggingface/transformers.git@3b39b906183ed08d9961908eb73104aeea345d11
+typing_extensions==4.7.1
+tzdata==2023.3
+urllib3==2.0.4
+uvicorn==0.23.2
+vigogne @ git+https://github.com/bofenghuang/vigogne@76e1cd0b35fd4f9e360aecdf7130c998459df0ff
+wandb==0.15.9
+websockets==11.0.3
+xxhash==3.3.0
+yarl==1.9.2