lisa-on-cuda / app.py
alessandro trinca tornidor
[refactor] start reverting to original app.py content/1
a84a5a1
raw
history blame
3.5 kB
import argparse
import cv2
import gradio as gr
import json
import logging
import nh3
import numpy as np
import os
import re
import sys
import torch
import torch.nn.functional as F
from fastapi import FastAPI, File, UploadFile, Request
from fastapi.responses import HTMLResponse, RedirectResponse
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from transformers import AutoTokenizer, BitsAndBytesConfig, CLIPImageProcessor
from typing import Callable
from model.LISA import LISAForCausalLM
from model.llava import conversation as conversation_lib
from model.llava.mm_utils import tokenizer_image_token
from model.segment_anything.utils.transforms import ResizeLongestSide
from utils import session_logger
from utils.utils import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN,
DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX)
session_logger.change_logging(logging.DEBUG)
CUSTOM_GRADIO_PATH = "/"
app = FastAPI(title="lisa_app", version="1.0")
FASTAPI_STATIC = os.getenv("FASTAPI_STATIC")
os.makedirs(FASTAPI_STATIC, exist_ok=True)
app.mount("/static", StaticFiles(directory=FASTAPI_STATIC), name="static")
templates = Jinja2Templates(directory="templates")
@app.get("/health")
@session_logger.set_uuid_logging
def health() -> str:
try:
logging.info("health check")
return json.dumps({"msg": "ok"})
except Exception as e:
logging.error(f"exception:{e}.")
return json.dumps({"msg": "request failed"})
@session_logger.set_uuid_logging
def get_cleaned_input(input_str):
logging.info(f"start cleaning of input_str: {input_str}.")
input_str = nh3.clean(
input_str,
tags={
"a",
"abbr",
"acronym",
"b",
"blockquote",
"code",
"em",
"i",
"li",
"ol",
"strong",
"ul",
},
attributes={
"a": {"href", "title"},
"abbr": {"title"},
"acronym": {"title"},
},
url_schemes={"http", "https", "mailto"},
link_rel=None,
)
logging.info(f"cleaned input_str: {input_str}.")
return input_str
@session_logger.set_uuid_logging
def get_inference_model_by_args(args_to_parse):
logging.info(f"args_to_parse:{args_to_parse}.")
@session_logger.set_uuid_logging
def inference(input_str, input_image):
## filter out special chars
input_str = get_cleaned_input(input_str)
logging.info(f"input_str type: {type(input_str)}, input_image type: {type(input_image)}.")
logging.info(f"input_str: {input_str}.")
return output_image, output_str
return inference
@session_logger.set_uuid_logging
def get_gradio_interface(fn_inference: Callable):
return gr.Interface(
fn_inference,
inputs=[
gr.Textbox(lines=1, placeholder=None, label="Text Instruction"),
gr.Image(type="filepath", label="Input Image")
],
outputs=[
gr.Image(type="pil", label="Segmentation Output"),
gr.Textbox(lines=1, placeholder=None, label="Text Output"),
],
title=title,
description=description,
article=article,
examples=examples,
allow_flagging="auto",
)
args = parse_args(sys.argv[1:])
inference_fn = get_inference_model_by_args(args)
io = get_gradio_interface(inference_fn)
app = gr.mount_gradio_app(app, io, path=CUSTOM_GRADIO_PATH)