Spaces:
Runtime error
Runtime error
File size: 654 Bytes
5bbb821 5eea5ad e13658e 5eea5ad 54e091e 5eea5ad 295f47e 5eea5ad 0690567 e13658e e5aa658 0690567 e13658e 0690567 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 |
import os
from fastapi import FastAPI, Request, Response
from langserve import APIHandler
from pydantic import BaseModel
from langchain_huggingface import HuggingFaceEndpoint
app = FastAPI()
@app.get("/")
def greet_json():
return {"Hello": "World!"}
llm = HuggingFaceEndpoint(
repo_id="meta-llama/Meta-Llama-3-8B-Instruct",
)
api_handler = APIHandler(llm, path="/v1")
# class InputRequest(BaseModel):
# input: str
@app.post("/v1/stream")
async def simple_stream(request: Request):
print(request)
return await api_handler.stream(request)
if __name__ == "__main__":
uvicorn.run("app:app", host="localhost", reload=True) |