working app

=0.26.0

@@ -1,39 +0,0 @@
-Collecting accelerate
-  Downloading accelerate-1.0.1-py3-none-any.whl (330 kB)
-     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 330.9/330.9 kB 10.6 MB/s eta 0:00:00
-Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/site-packages (from accelerate) (2.5.0)
-Requirement already satisfied: numpy<3.0.0,>=1.17 in /usr/local/lib/python3.10/site-packages (from accelerate) (2.1.2)
-Requirement already satisfied: huggingface-hub>=0.21.0 in /usr/local/lib/python3.10/site-packages (from accelerate) (0.26.0)
-Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/site-packages (from accelerate) (6.0.2)
-Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/site-packages (from accelerate) (24.1)
-Requirement already satisfied: safetensors>=0.4.3 in /usr/local/lib/python3.10/site-packages (from accelerate) (0.4.5)
-Requirement already satisfied: psutil in /usr/local/lib/python3.10/site-packages (from accelerate) (5.9.8)
-Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/site-packages (from huggingface-hub>=0.21.0->accelerate) (2024.6.1)
-Requirement already satisfied: requests in /usr/local/lib/python3.10/site-packages (from huggingface-hub>=0.21.0->accelerate) (2.32.3)
-Requirement already satisfied: filelock in /usr/local/lib/python3.10/site-packages (from huggingface-hub>=0.21.0->accelerate) (3.16.1)
-Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/site-packages (from huggingface-hub>=0.21.0->accelerate) (4.66.5)
-Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/site-packages (from huggingface-hub>=0.21.0->accelerate) (4.12.2)
-Requirement already satisfied: nvidia-cuda-runtime-cu12==12.4.127 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (12.4.127)
-Requirement already satisfied: triton==3.1.0 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.1.0)
-Requirement already satisfied: nvidia-cusolver-cu12==11.6.1.9 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (11.6.1.9)
-Requirement already satisfied: nvidia-nccl-cu12==2.21.5 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (2.21.5)
-Requirement already satisfied: nvidia-cusparse-cu12==12.3.1.170 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (12.3.1.170)
-Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.1.4)
-Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.4.127 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (12.4.127)
-Requirement already satisfied: nvidia-cuda-cupti-cu12==12.4.127 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (12.4.127)
-Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (9.1.0.70)
-Requirement already satisfied: nvidia-nvjitlink-cu12==12.4.127 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (12.4.127)
-Requirement already satisfied: networkx in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.4.1)
-Requirement already satisfied: nvidia-nvtx-cu12==12.4.127 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (12.4.127)
-Requirement already satisfied: nvidia-cublas-cu12==12.4.5.8 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (12.4.5.8)
-Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (1.13.1)
-Requirement already satisfied: nvidia-cufft-cu12==11.2.1.3 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (11.2.1.3)
-Requirement already satisfied: nvidia-curand-cu12==10.3.5.147 in /usr/local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (10.3.5.147)
-Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/site-packages (from sympy==1.13.1->torch>=1.10.0->accelerate) (1.3.0)
-Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.5)
-Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/site-packages (from requests->huggingface-hub>=0.21.0->accelerate) (2024.8.30)
-Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests->huggingface-hub>=0.21.0->accelerate) (3.4.0)
-Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/site-packages (from requests->huggingface-hub>=0.21.0->accelerate) (2.2.3)
-Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests->huggingface-hub>=0.21.0->accelerate) (3.10)
-Installing collected packages: accelerate
-Successfully installed accelerate-1.0.1

app.py

@@ -1,525 +1,135 @@
-import argparse
-import datetime
-import json
-import os
-import time
-
 import gradio as gr
-import requests
 from inference import inference_and_run
-from conversation import (default_conversation, conv_templates,
-                                   SeparatorStyle)
-
-LOGDIR = "."
-from utils import (build_logger, server_error_msg,
-                            violates_moderation, moderation_msg)
-import hashlib
 import spaces
+import os
+import re
+import shutil
 
-logger = build_logger("gradio_web_server", "gradio_web_server.log")
-
-headers = {"User-Agent": "LLaVA Client"}
-
-no_change_btn = gr.Button()
-enable_btn = gr.Button(interactive=True)
-disable_btn = gr.Button(interactive=False)
-
-priority = {
-    "vicuna-13b": "aaaaaaa",
-    "koala-13b": "aaaaaab",
-}
-
-
-def get_conv_log_filename():
-    t = datetime.datetime.now()
-    name = os.path.join(LOGDIR, f"{t.year}-{t.month:02d}-{t.day:02d}-conv.json")
-    return name
-
-
-def get_model_list():
-    # ret = requests.post(args.controller_url + "/refresh_all_workers")
-    # assert ret.status_code == 200
-    # ret = requests.post(args.controller_url + "/list_models")
-    # models = ret.json()["models"]
-    # models.sort(key=lambda x: priority.get(x, x))
-    # logger.info(f"Models: {models}")
-    # return models
-    models = ["jadechoghari/Ferret-UI-Gemma2b"]
-    logger.info(f"Models: {models}")
-    return models
-
-get_window_url_params = """
-function() {
-    const params = new URLSearchParams(window.location.search);
-    url_params = Object.fromEntries(params);
-    console.log(url_params);
-    return url_params;
-    }
-"""
-
-
-def load_demo(url_params, request: gr.Request):
-
-    dropdown_update = gr.Dropdown(visible=True)
-    if "model" in url_params:
-        model = url_params["model"]
-        if model in models:
-            dropdown_update = gr.Dropdown(value=model, visible=True)
-
-    state = default_conversation.copy()
-    return state, dropdown_update
-
-
-def load_demo_refresh_model_list(request: gr.Request):
-    models = get_model_list()
-    state = default_conversation.copy()
-    dropdown_update = gr.Dropdown(
-        choices=models,
-        value=models[0] if len(models) > 0 else ""
-    )
-    return state, dropdown_update
-
-
-def vote_last_response(state, vote_type, model_selector, request: gr.Request):
-    with open(get_conv_log_filename(), "a") as fout:
-        data = {
-            "tstamp": round(time.time(), 4),
-            "type": vote_type,
-            "model": model_selector,
-            "state": state.dict(),
-            "ip": request.client.host,
-        }
-        fout.write(json.dumps(data) + "\n")
-
-
-def upvote_last_response(state, model_selector, request: gr.Request):
-    vote_last_response(state, "upvote", model_selector, request)
-    return ("",) + (disable_btn,) * 3
-
-
-def downvote_last_response(state, model_selector, request: gr.Request):
-    vote_last_response(state, "downvote", model_selector, request)
-    return ("",) + (disable_btn,) * 3
-
-
-def flag_last_response(state, model_selector, request: gr.Request):
-    vote_last_response(state, "flag", model_selector, request)
-    return ("",) + (disable_btn,) * 3
-
-
-def regenerate(state, image_process_mode, request: gr.Request):
-    state.messages[-1][-1] = None
-    prev_human_msg = state.messages[-2]
-    if type(prev_human_msg[1]) in (tuple, list):
-        prev_human_msg[1] = (*prev_human_msg[1][:2], image_process_mode)
-    state.skip_next = False
-    return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 5
-
-
-def clear_history(request: gr.Request):
-    state = default_conversation.copy()
-    return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 5
-
-
-def add_text(state, text, image, image_process_mode, request: gr.Request):
-    if len(text) <= 0 and image is None:
-        state.skip_next = True
-        return (state, state.to_gradio_chatbot(), "", None) + (no_change_btn,) * 5
-    if args.moderate:
-        flagged = violates_moderation(text)
-        if flagged:
-            state.skip_next = True
-            return (state, state.to_gradio_chatbot(), moderation_msg, None) + (
-                no_change_btn,) * 5
-
-    text = text[:1536]  # Hard cut-off
-    if image is not None:
-        text = text[:1200]  # Hard cut-off for images
-        if '<image>' not in text:
-            # text = '<Image><image></Image>' + text
-            text = text + '\n<image>'
-        text = (text, image, image_process_mode)
-        state = default_conversation.copy()
-    state.append_message(state.roles[0], text)
-    state.append_message(state.roles[1], None)
-    state.skip_next = False
-    return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 5
+model_name = 'Ferret-UI'
+cur_dir = os.path.dirname(os.path.abspath(__file__))
 
 @spaces.GPU()
-def http_bot(state, model_selector, temperature, top_p, max_new_tokens, request: gr.Request):
-    start_tstamp = time.time()
-    model_name = model_selector
-
-    if state.skip_next:
-        # This generate call is skipped due to invalid inputs
-        yield (state, state.to_gradio_chatbot()) + (no_change_btn,) * 5
-        return
-
-    if len(state.messages) == state.offset + 2:
-        # First round of conversation
-        if "llava" in model_name.lower():
-            if 'llama-2' in model_name.lower():
-                template_name = "llava_llama_2"
-            elif "mistral" in model_name.lower() or "mixtral" in model_name.lower():
-                if 'orca' in model_name.lower():
-                    template_name = "mistral_orca"
-                elif 'hermes' in model_name.lower():
-                    template_name = "chatml_direct"
-                else:
-                    template_name = "mistral_instruct"
-            elif 'llava-v1.6-34b' in model_name.lower():
-                template_name = "chatml_direct"
-            elif "v1" in model_name.lower():
-                if 'mmtag' in model_name.lower():
-                    template_name = "v1_mmtag"
-                elif 'plain' in model_name.lower() and 'finetune' not in model_name.lower():
-                    template_name = "v1_mmtag"
-                else:
-                    template_name = "llava_v1"
-            elif "mpt" in model_name.lower():
-                template_name = "mpt"
-            else:
-                if 'mmtag' in model_name.lower():
-                    template_name = "v0_mmtag"
-                elif 'plain' in model_name.lower() and 'finetune' not in model_name.lower():
-                    template_name = "v0_mmtag"
-                else:
-                    template_name = "llava_v0"
-        elif "mpt" in model_name:
-            template_name = "mpt_text"
-        elif "llama-2" in model_name:
-            template_name = "llama_2"
-        elif "gemma" in model_name.lower():
-          template_name = "ferret_gemma_instruct"
-          print("conv mode to gemma")
-        else:
-            template_name = "vicuna_v1"
-        new_state = conv_templates[template_name].copy()
-        new_state.append_message(new_state.roles[0], state.messages[-2][1])
-        new_state.append_message(new_state.roles[1], None)
-        state = new_state
-
-    # # Query worker address
-    # controller_url = args.controller_url
-    # ret = requests.post(controller_url + "/get_worker_address",
-    #         json={"model": model_name})
-    # worker_addr = ret.json()["address"]
-    # logger.info(f"model_name: {model_name}, worker_addr: {worker_addr}")
-
-  
-
-    # # No available worker
-    # if worker_addr == "":
-    #     state.messages[-1][-1] = server_error_msg
-    #     yield (state, state.to_gradio_chatbot(), disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
-    #     return
-
-    # Construct prompt
-    prompt = state.get_prompt()
+def inference_with_gradio(chatbot, image, prompt, model_path, box=None):
+    dir_path = os.path.dirname(image)
+    # image_path = image
+    # Define the directory where you want to save the image (current directory)
+    filename = os.path.basename(image)
     dir_path = "./"
-    all_images = state.get_images(return_pil=True)
-    all_image_hash = [hashlib.md5(image.tobytes()).hexdigest() for image in all_images]
-    for image, hash in zip(all_images, all_image_hash):
-        t = datetime.datetime.now()
-        # dir_path = os.path.join(LOGDIR, "serve_images", f"{t.year}-{t.month:02d}-{t.day:02d}")
-        # filename = os.path.join(LOGDIR, "serve_images", f"{t.year}-{t.month:02d}-{t.day:02d}", f"{hash}.jpg")
-        # filename = os.path.join(dir_path, f"{hash}.jpg")
-        filename = os.path.join(dir_path, f"{hash}.jpg")
-        if not os.path.isfile(filename):
-            os.makedirs(os.path.dirname(filename), exist_ok=True)
-            image.save(filename)
-
-    # Make requests
-    pload = {
-        "model": model_name,
-        "prompt": prompt,
-        "temperature": float(temperature),
-        "top_p": float(top_p),
-        "max_new_tokens": min(int(max_new_tokens), 1536),
-        "stop": state.sep if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT] else state.sep2,
-        "images": f'List of {len(state.get_images())} images: {all_image_hash}',
-    }
-    logger.info(f"==== request ====\n{pload}")
-
-    pload['images'] = state.get_images()
 
-    state.messages[-1][-1] = "▌"
-    yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
-
-    try:
-        # Stream output
-        # response = requests.post(worker_addr + "/worker_generate_stream",
-        #     headers=headers, json=pload, stream=True, timeout=10)
-        stop = state.sep if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT] else state.sep2
-        #TODO: define inference and run function
-        extracted_texts = inference_and_run(
+    # Create the new path for the file (in the current directory)
+    image_path = os.path.join(dir_path, filename)
+    shutil.copy(image, image_path)
+    print("filename path: ", filename)
+    if "gemma" in model_path.lower():
+        conv_mode = "ferret_gemma_instruct"
+    else:
+        conv_mode = "ferret_llama_3"
+    
+    # inference_text = inference_and_run(
+    #     image_path=image_path,
+    #     prompt=prompt,
+    #     conv_mode=conv_mode,
+    #     model_path=model_path,
+    #     box=box
+    # )
+    inference_text = inference_and_run(
         image_path=filename, # double check this
         image_dir=dir_path,
         prompt=prompt,
-        model_path=model_name,
-        conv_mode="ferret_gemma_instruct",  # Default mode from the original function
-        temperature=temperature, 
-        top_p=top_p,
-        max_new_tokens=max_new_tokens,
-        stop=stop    # Assuming we want to process the image
+        model_path="jadechoghari/Ferret-UI-Gemma2b",
+        conv_mode=conv_mode,  # Default mode from the original function
+        # temperature=temperature, 
+        # top_p=top_p,
+        # max_new_tokens=max_new_tokens,
+        # stop=stop    # Assuming we want to process the image
         )
-        response = extracted_texts
-        logger.info(f"This is the respone {response}")
-        delay=0.01
-        # for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"):
-        #     if chunk:
-        #         data = json.loads(chunk.decode())
-        #         if data["error_code"] == 0:
-        #             output = data["text"][len(prompt):].strip()
-        #             state.messages[-1][-1] = output + "▌"
-        #             yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
-        #         else:
-        #             output = data["text"] + f" (error_code: {data['error_code']})"
-        #             state.messages[-1][-1] = output
-        #             yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
-        #             return
-        #         time.sleep(0.03)
-        text = response[0]
-        output = ""  # Will hold the progressively built output
-        for i, char in enumerate(text):
-            output += char
-            state.messages[-1][-1] = output + "▌"  # Add cursor ▌ at the end
-            yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
-            time.sleep(delay)  # Control typing speed with delay
-    except requests.exceptions.RequestException as e:
-        state.messages[-1][-1] = server_error_msg
-        yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
-        return
-
-    state.messages[-1][-1] = state.messages[-1][-1][:-1]
-    yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5
-
-    finish_tstamp = time.time()
-    logger.info(f"{output}")
-
-    with open(get_conv_log_filename(), "a") as fout:
-        data = {
-            "tstamp": round(finish_tstamp, 4),
-            "type": "chat",
-            "model": model_name,
-            "start": round(start_tstamp, 4),
-            "finish": round(finish_tstamp, 4),
-            "state": state.dict(),
-            "images": all_image_hash,
-            "ip": request.client.host,
-        }
-        fout.write(json.dumps(data) + "\n")
-
-title_markdown = ("""
-# 🌋 LLaVA: Large Language and Vision Assistant
-[[Project Page](https://llava-vl.github.io)] [[Code](https://github.com/haotian-liu/LLaVA)] [[Model](https://github.com/haotian-liu/LLaVA/blob/main/docs/MODEL_ZOO.md)] | 📚 [[LLaVA](https://arxiv.org/abs/2304.08485)] [[LLaVA-v1.5](https://arxiv.org/abs/2310.03744)] [[LLaVA-v1.6](https://llava-vl.github.io/blog/2024-01-30-llava-1-6/)]
-""")
-
-tos_markdown = ("""
-### Terms of use
-By using this service, users are required to agree to the following terms:
-The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research.
-Please click the "Flag" button if you get any inappropriate answer! We will collect those to keep improving our moderator.
-For an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.
-""")
-
-
-learn_more_markdown = ("""
-### License
-The service is a research preview intended for non-commercial use only, subject to the model [License](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) of LLaMA, [Terms of Use](https://openai.com/policies/terms-of-use) of the data generated by OpenAI, and [Privacy Practices](https://chrome.google.com/webstore/detail/sharegpt-share-your-chatg/daiacboceoaocpibfodeljbdfacokfjb) of ShareGPT. Please contact us if you find any potential violation.
-""")
-
-block_css = """
-
-#buttons button {
-    min-width: min(120px,100%);
-}
-
+    
+    # print("done, now appending", inference_text)
+    # chatbot.append((prompt, inference_text))
+    # return chatbot
+    # Convert inference_text to string if it's not already
+    if isinstance(inference_text, (list, tuple)):
+        inference_text = str(inference_text[0])
+        
+    # Update chatbot history with new message pair
+    new_history = chatbot.copy() if chatbot else []
+    new_history.append((prompt, inference_text))
+    return new_history
+
+def submit_chat(chatbot, text_input):
+    response = ''
+    chatbot.append((text_input, response))
+    return chatbot, ''
+
+def clear_chat():
+    return [], None, ""
+
+with open(f"{cur_dir}/logo.svg", "r", encoding="utf-8") as svg_file:
+    svg_content = svg_file.read()
+font_size = "2.5em"
+svg_content = re.sub(r'(<svg[^>]*)(>)', rf'\1 height="{font_size}" style="vertical-align: middle; display: inline-block;"\2', svg_content)
+html = f"""
+<p align="center" style="font-size: {font_size}; line-height: 1;">
+    <span style="display: inline-block; vertical-align: middle;">{svg_content}</span>
+    <span style="display: inline-block; vertical-align: middle;">{model_name}</span>
+</p>
+<center><font size=3><b>{model_name}</b> Demo: Upload an image, provide a prompt, and get insights using advanced AI models. <a href='https://huggingface.co/jadechoghari/Ferret-UI-Gemma2b'>😊 Huggingface</a></font></center>
 """
 
-def build_demo(embed_mode, cur_dir=None, concurrency_count=10):
-    textbox = gr.Textbox(show_label=False, placeholder="Enter text and press ENTER", container=False)
-    with gr.Blocks(title="LLaVA", theme=gr.themes.Default(), css=block_css) as demo:
-        state = gr.State()
-
-        if not embed_mode:
-            gr.Markdown(title_markdown)
-
-        with gr.Row():
-            models = [
-              "jadechoghari/Ferret-UI-Gemma2b"
-            ]
-            with gr.Column(scale=3):
-                with gr.Row(elem_id="model_selector_row"):
-                    model_selector = gr.Dropdown(
-                        choices=models,
-                        value=models[0] if len(models) > 0 else "",
-                        interactive=True,
-                        show_label=False,
-                        container=False)
-
-                imagebox = gr.Image(type="pil")
-                image_process_mode = gr.Radio(
-                    ["Crop", "Resize", "Pad", "Default"],
-                    value="Default",
-                    label="Preprocess for non-square image", visible=False)
-
-                if cur_dir is None:
-                    cur_dir = os.path.dirname(os.path.abspath(__file__))
-                gr.Examples(examples=[
-                    [f"{cur_dir}/examples/extreme_ironing.jpg", "What is unusual about this image?"],
-                    [f"{cur_dir}/examples/waterview.jpg", "What are the things I should be cautious about when I visit here?"],
-                ], inputs=[imagebox, textbox])
-
-                with gr.Accordion("Parameters", open=False) as parameter_row:
-                    temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.2, step=0.1, interactive=True, label="Temperature",)
-                    top_p = gr.Slider(minimum=0.0, maximum=1.0, value=0.7, step=0.1, interactive=True, label="Top P",)
-                    max_output_tokens = gr.Slider(minimum=0, maximum=1024, value=512, step=64, interactive=True, label="Max output tokens",)
-
-            with gr.Column(scale=8):
-                chatbot = gr.Chatbot(
-                    elem_id="chatbot",
-                    label="LLaVA Chatbot",
-                    height=650,
-                    layout="panel",
-                )
-                with gr.Row():
-                    with gr.Column(scale=8):
-                        textbox.render()
-                    with gr.Column(scale=1, min_width=50):
-                        submit_btn = gr.Button(value="Send", variant="primary")
-                with gr.Row(elem_id="buttons") as button_row:
-                    upvote_btn = gr.Button(value="👍  Upvote", interactive=False)
-                    downvote_btn = gr.Button(value="👎  Downvote", interactive=False)
-                    flag_btn = gr.Button(value="⚠️  Flag", interactive=False)
-                    #stop_btn = gr.Button(value="⏹️  Stop Generation", interactive=False)
-                    regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False)
-                    clear_btn = gr.Button(value="🗑️  Clear", interactive=False)
-
-        if not embed_mode:
-            gr.Markdown(tos_markdown)
-            gr.Markdown(learn_more_markdown)
-        url_params = gr.JSON(visible=False)
-
-        # Register listeners
-        btn_list = [upvote_btn, downvote_btn, flag_btn, regenerate_btn, clear_btn]
-        upvote_btn.click(
-            upvote_last_response,
-            [state, model_selector],
-            [textbox, upvote_btn, downvote_btn, flag_btn]
-        )
-        downvote_btn.click(
-            downvote_last_response,
-            [state, model_selector],
-            [textbox, upvote_btn, downvote_btn, flag_btn]
-        )
-        flag_btn.click(
-            flag_last_response,
-            [state, model_selector],
-            [textbox, upvote_btn, downvote_btn, flag_btn]
-        )
-
-        regenerate_btn.click(
-            regenerate,
-            [state, image_process_mode],
-            [state, chatbot, textbox, imagebox] + btn_list
-        ).then(
-            http_bot,
-            [state, model_selector, temperature, top_p, max_output_tokens],
-            [state, chatbot] + btn_list,
-            concurrency_limit=concurrency_count
-        )
-
-        clear_btn.click(
-            clear_history,
-            None,
-            [state, chatbot, textbox, imagebox] + btn_list,
-            queue=False
-        )
-
-        textbox.submit(
-            add_text,
-            [state, textbox, imagebox, image_process_mode],
-            [state, chatbot, textbox, imagebox] + btn_list,
-            queue=False
-        ).then(
-            http_bot,
-            [state, model_selector, temperature, top_p, max_output_tokens],
-            [state, chatbot] + btn_list,
-            concurrency_limit=concurrency_count
-        )
-
-        submit_btn.click(
-            add_text,
-            [state, textbox, imagebox, image_process_mode],
-            [state, chatbot, textbox, imagebox] + btn_list
-        ).then(
-            http_bot,
-            [state, model_selector, temperature, top_p, max_output_tokens],
-            [state, chatbot] + btn_list,
-            concurrency_limit=concurrency_count
-        )
-
-        if args.model_list_mode == "once":
-            demo.load(
-                load_demo,
-                [url_params],
-                [state, model_selector],
-                js=get_window_url_params
-            )
-        elif args.model_list_mode == "reload":
-            demo.load(
-                load_demo_refresh_model_list,
-                None,
-                [state, model_selector],
-                queue=False
-            )
-        else:
-            raise ValueError(f"Unknown model list mode: {args.model_list_mode}")
-
-    return demo
-
-
-# if __name__ == "__main__":
-#     parser = argparse.ArgumentParser()
-#     parser.add_argument("--port", type=int, default=7860)  # You can still specify the port
-#     parser.add_argument("--controller-url", type=str, default="http://localhost:21001")
-#     parser.add_argument("--concurrency-count", type=int, default=16)
-#     parser.add_argument("--model-list-mode", type=str, default="once", choices=["once", "reload"])
-#     parser.add_argument("--share", action="store_true")
-#     parser.add_argument("--moderate", action="store_true")
-#     parser.add_argument("--embed", action="store_true")
-#     args = parser.parse_args()
-#     # models = get_model_list()
-#     demo = build_demo(args.embed, concurrency_count=args.concurrency_count)
-#     demo.queue(api_open=False).launch(
-#         server_port=args.port,  # Specify the port if needed
-#         share=True, 
-#         debug=True        # All other functionalities like sharing still work
-#     )
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--host", type=str, default="0.0.0.0")
-    parser.add_argument("--port", type=int)
-    parser.add_argument("--controller-url", type=str, default="http://localhost:21001")
-    parser.add_argument("--concurrency-count", type=int, default=16)
-    parser.add_argument("--model-list-mode", type=str, default="once",
-        choices=["once", "reload"])
-    parser.add_argument("--share", action="store_true")
-    parser.add_argument("--moderate", action="store_true")
-    parser.add_argument("--embed", action="store_true")
-    args = parser.parse_args()
-    logger.info(f"args: {args}")
-
-    models = get_model_list()
-
-    logger.info(args)
-    demo = build_demo(args.embed, concurrency_count=args.concurrency_count)
-    demo.queue(
-        api_open=False
-    ).launch(
-        server_name=args.host,
-        server_port=args.port,
-        share=True,
-        debug=True
-    )
+latex_delimiters_set = [{
+        "left": "\\(",
+        "right": "\\)",
+        "display": False 
+    }, {
+        "left": "\\begin{equation}",
+        "right": "\\end{equation}",
+        "display": True 
+    }, {
+        "left": "\\begin{align}",
+        "right": "\\end{align}",
+        "display": True
+    }]
+
+# Set up UI components
+image_input = gr.Image(label="Upload Image", type="filepath", height=350)
+text_input = gr.Textbox(lines=2, placeholder="Enter your prompt here...", label="Prompt")
+model_dropdown = gr.Dropdown(choices=[
+    "jadechoghari/Ferret-UI-Gemma2b",
+    "jadechoghari/Ferret-UI-Llama8b",
+], label="Model Path", value="jadechoghari/Ferret-UI-Gemma2b")
+
+bounding_box_input = gr.Textbox(placeholder="Optional bounding box (x1, y1, x2, y2)", label="Bounding Box (optional)")
+chatbot = gr.Chatbot(label="Chat with Ferret-UI", height=400, show_copy_button=True, latex_delimiters=latex_delimiters_set)
+
+with gr.Blocks(title=model_name, theme=gr.themes.Ocean()) as demo:
+    gr.HTML(html)
+    with gr.Row():
+        with gr.Column(scale=3):
+            # gr.Examples(
+            #     examples=[
+            #         ["appstore_reminders.png", "Describe the image in details", "jadechoghari/Ferret-UI-Gemma2b", None],
+            #         ["appstore_reminders.png", "What's inside the selected region?", "jadechoghari/Ferret-UI-Gemma2b", "189, 906, 404, 970"],
+            #         ["appstore_reminders.png", "Where is the Game Tab?", "jadechoghari/Ferret-UI-Gemma2b", None],
+            #     ],
+            #     inputs=[image_input, text_input, model_dropdown, bounding_box_input]
+            # )
+            image_input.render()
+            text_input.render()
+            model_dropdown.render()
+            bounding_box_input.render()
+        with gr.Column(scale=7):
+            chatbot.render()
+            with gr.Row():
+                send_btn = gr.Button("Send", variant="primary")
+                clear_btn = gr.Button("Clear", variant="secondary")
+
+    send_click_event = send_btn.click(
+        inference_with_gradio, [chatbot, image_input, text_input, model_dropdown, bounding_box_input], chatbot
+    ).then(submit_chat, [chatbot, text_input], [chatbot, text_input])
+    submit_event = text_input.submit(
+        inference_with_gradio, [chatbot, image_input, text_input, model_dropdown, bounding_box_input], chatbot
+    ).then(submit_chat, [chatbot, text_input], [chatbot, text_input])
+    
+    clear_btn.click(clear_chat, outputs=[chatbot, image_input, text_input, bounding_box_input])
+
+demo.launch()

app.pyi

@@ -1,797 +0,0 @@
-'''
-Usage:
-
-python -m ferret.serve.gradio_web_server --controller http://localhost:10000 --add_region_feature
-'''
-import argparse
-import datetime
-import json
-import os
-import time
-
-import gradio as gr
-import requests
-
-from conversation import (default_conversation, conv_templates,
-                                   SeparatorStyle)
-from constants import LOGDIR
-from utils import (build_logger, server_error_msg,
-    violates_moderation, moderation_msg)
-import hashlib
-# Added 
-import re
-from copy import deepcopy
-from PIL import ImageDraw, ImageFont
-from gradio import processing_utils
-import numpy as np
-import torch
-import torch.nn.functional as F
-from scipy.ndimage import binary_dilation, binary_erosion
-import pdb
-from gradio_css import code_highlight_css
-import spaces
-
-from inference import inference_and_run
-
-DEFAULT_REGION_REFER_TOKEN = "[region]"
-DEFAULT_REGION_FEA_TOKEN = "<region_fea>"
-
-
-logger = build_logger("gradio_web_server", "gradio_web_server.log")
-
-headers = {"User-Agent": "FERRET Client"}
-
-no_change_btn = gr.Button
-enable_btn = gr.Button(interactive=True)
-disable_btn = gr.Button(interactive=False)
-
-priority = {
-    "vicuna-13b": "aaaaaaa",
-    "koala-13b": "aaaaaab",
-}
-
-VOCAB_IMAGE_W = 1000  # 224
-VOCAB_IMAGE_H = 1000  # 224
-
-def generate_mask_for_feature(coor, raw_w, raw_h, mask=None):
-    if mask is not None:
-        assert mask.shape[0] == raw_w and mask.shape[1] == raw_h
-    coor_mask = torch.zeros((raw_w, raw_h))
-    # Assume it samples a point.
-    if len(coor) == 2:
-        # Define window size
-        span = 5
-        # Make sure the window does not exceed array bounds
-        x_min = max(0, coor[0] - span)
-        x_max = min(raw_w, coor[0] + span + 1)
-        y_min = max(0, coor[1] - span)
-        y_max = min(raw_h, coor[1] + span + 1)
-        coor_mask[int(x_min):int(x_max), int(y_min):int(y_max)] = 1
-        assert (coor_mask==1).any(), f"coor: {coor}, raw_w: {raw_w}, raw_h: {raw_h}"
-    elif len(coor) == 4:
-        # Box input or Sketch input.
-        coor_mask = torch.zeros((raw_w, raw_h))
-        coor_mask[coor[0]:coor[2]+1, coor[1]:coor[3]+1] = 1
-        if mask is not None:
-            coor_mask = coor_mask * mask
-    # coor_mask = torch.from_numpy(coor_mask)
-    # pdb.set_trace()
-    assert len(coor_mask.nonzero()) != 0
-    return coor_mask.tolist()
-
-
-def draw_box(coor, region_mask, region_ph, img, input_mode):
-    colors = ["red"]
-    draw = ImageDraw.Draw(img)
-    font = ImageFont.truetype("./DejaVuSans.ttf", size=18)
-    if input_mode == 'Box':
-        draw.rectangle([coor[0], coor[1], coor[2], coor[3]], outline=colors[0], width=4)
-        draw.rectangle([coor[0], coor[3] - int(font.size * 1.2), coor[0] + int((len(region_ph) + 0.8) * font.size * 0.6), coor[3]], outline=colors[0], fill=colors[0], width=4)
-        draw.text([coor[0] + int(font.size * 0.2), coor[3] - int(font.size*1.2)], region_ph, font=font, fill=(255,255,255))
-    elif input_mode == 'Point':
-        r = 8 
-        leftUpPoint = (coor[0]-r, coor[1]-r)
-        rightDownPoint = (coor[0]+r, coor[1]+r)
-        twoPointList = [leftUpPoint, rightDownPoint]
-        draw.ellipse(twoPointList, outline=colors[0], width=4)
-        draw.rectangle([coor[0], coor[1], coor[0] + int((len(region_ph) + 0.8) * font.size * 0.6), coor[1] + int(font.size * 1.2)], outline=colors[0], fill=colors[0], width=4)
-        draw.text([coor[0] + int(font.size * 0.2), coor[1]], region_ph, font=font, fill=(255,255,255))
-    elif input_mode == 'Sketch':
-        draw.rectangle([coor[0], coor[3] - int(font.size * 1.2), coor[0] + int((len(region_ph) + 0.8) * font.size * 0.6), coor[3]], outline=colors[0], fill=colors[0], width=4)
-        draw.text([coor[0] + int(font.size * 0.2), coor[3] - int(font.size*1.2)], region_ph, font=font, fill=(255,255,255))
-        # Use morphological operations to find the boundary
-        mask = np.array(region_mask)
-        dilated = binary_dilation(mask, structure=np.ones((3,3)))
-        eroded = binary_erosion(mask, structure=np.ones((3,3)))
-        boundary = dilated ^ eroded  # XOR operation to find the difference between dilated and eroded mask
-        # Loop over the boundary and paint the corresponding pixels
-        for i in range(boundary.shape[0]):
-            for j in range(boundary.shape[1]):
-                if boundary[i, j]:
-                    # This is a pixel on the boundary, paint it red
-                    draw.point((i, j), fill=colors[0])
-    else:
-        NotImplementedError(f'Input mode of {input_mode} is not Implemented.')
-    return img
-
-
-def get_conv_log_filename():
-    t = datetime.datetime.now()
-    name = os.path.join(LOGDIR, f"{t.year}-{t.month:02d}-{t.day:02d}-conv.json")
-    return name
-
-
-# TODO: return model manually just one for now called "jadechoghari/Ferret-UI-Gemma2b"
-def get_model_list():
-    # ret = requests.post(args.controller_url + "/refresh_all_workers")
-    # assert ret.status_code == 200
-    # ret = requests.post(args.controller_url + "/list_models")
-    # models = ret.json()["models"]
-    # models.sort(key=lambda x: priority.get(x, x))
-    # logger.info(f"Models: {models}")
-    # return models
-    models = ["jadechoghari/Ferret-UI-Gemma2b"]
-    logger.info(f"Models: {models}")
-    return models
-
-
-get_window_url_params = """
-function() {
-    const params = new URLSearchParams(window.location.search);
-    url_params = Object.fromEntries(params);
-    console.log(url_params);
-    return url_params;
-    }
-"""
-
-
-def load_demo(url_params, request: gr.Request):
-    # logger.info(f"load_demo. ip: {request.client.host}. params: {url_params}")
-
-    dropdown_update = gr.Dropdown(visible=True)
-    if "model" in url_params:
-        model = url_params["model"]
-        if model in models:
-            dropdown_update = gr.Dropdown(
-                value=model, visible=True)
-
-    state = default_conversation.copy()
-    print("state", state)
-    return (state,
-            dropdown_update,
-            gr.Chatbot(visible=True),
-            gr.Textbox(visible=True),
-            gr.Button(visible=True),
-            gr.Row(visible=True),
-            gr.Accordion(visible=True))
-
-
-def load_demo_refresh_model_list(request: gr.Request):
-    # logger.info(f"load_demo. ip: {request.client.host}")
-    models = get_model_list()
-    state = default_conversation.copy()
-    return (state, gr.Dropdown(
-               choices=models,
-               value=models[0] if len(models) > 0 else ""),
-            gr.Chatbot(visible=True),
-            gr.Textbox(visible=True),
-            gr.Button(visible=True),
-            gr.Row(visible=True),
-            gr.Accordion(visible=True))
-
-
-def vote_last_response(state, vote_type, model_selector, request: gr.Request):
-    with open(get_conv_log_filename(), "a") as fout:
-        data = {
-            "tstamp": round(time.time(), 4),
-            "type": vote_type,
-            "model": model_selector,
-            "state": state.dict(),
-            "ip": request.client.host,
-        }
-        fout.write(json.dumps(data) + "\n")
-
-
-def upvote_last_response(state, model_selector, request: gr.Request):
-    vote_last_response(state, "upvote", model_selector, request)
-    return ("",) + (disable_btn,) * 3
-
-
-def downvote_last_response(state, model_selector, request: gr.Request):
-    vote_last_response(state, "downvote", model_selector, request)
-    return ("",) + (disable_btn,) * 3
-
-
-def flag_last_response(state, model_selector, request: gr.Request):
-    vote_last_response(state, "flag", model_selector, request)
-    return ("",) + (disable_btn,) * 3
-
-
-def regenerate(state, image_process_mode, request: gr.Request):
-    state.messages[-1][-1] = None
-    prev_human_msg = state.messages[-2]
-    if type(prev_human_msg[1]) in (tuple, list):
-        prev_human_msg[1] = (*prev_human_msg[1][:2], image_process_mode)
-    state.skip_next = False
-    return (state, state.to_gradio_chatbot(), "") + (disable_btn,) * 5
-
-
-def clear_history(request: gr.Request):
-    state = default_conversation.copy()
-    return (state, state.to_gradio_chatbot(), "", None, None) + (disable_btn,) * 5 + \
-        (None, {'region_placeholder_tokens':[],'region_coordinates':[],'region_masks':[],'region_masks_in_prompts':[],'masks':[]}, [], None)
-
-
-def resize_bbox(box, image_w=None, image_h=None, default_wh=VOCAB_IMAGE_W):
-    ratio_w = image_w * 1.0 / default_wh
-    ratio_h = image_h * 1.0 / default_wh
-
-    new_box = [int(box[0] * ratio_w), int(box[1] * ratio_h), \
-               int(box[2] * ratio_w), int(box[3] * ratio_h)]
-    return new_box
-
-
-def show_location(sketch_pad, chatbot):
-    image = sketch_pad['image']
-    img_w, img_h = image.size
-    new_bboxes = []
-    old_bboxes = []
-    # chatbot[0] is image.
-    text = chatbot[1:]
-    for round_i in text:
-        human_input = round_i[0]
-        model_output = round_i[1]
-        # TODO: Difference: vocab representation.
-        # pattern = r'\[x\d*=(\d+(?:\.\d+)?), y\d*=(\d+(?:\.\d+)?), x\d*=(\d+(?:\.\d+)?), y\d*=(\d+(?:\.\d+)?)\]'
-        pattern = r'\[(\d+(?:\.\d+)?), (\d+(?:\.\d+)?), (\d+(?:\.\d+)?), (\d+(?:\.\d+)?)\]'
-        matches = re.findall(pattern, model_output)
-        for match in matches:
-            x1, y1, x2, y2 = map(int, match)
-            new_box = resize_bbox([x1, y1, x2, y2], img_w, img_h)
-            new_bboxes.append(new_box)
-            old_bboxes.append([x1, y1, x2, y2])
-        
-    set_old_bboxes = sorted(set(map(tuple, old_bboxes)), key=list(map(tuple, old_bboxes)).index)
-    list_old_bboxes = list(map(list, set_old_bboxes))
-
-    set_bboxes = sorted(set(map(tuple, new_bboxes)), key=list(map(tuple, new_bboxes)).index)
-    list_bboxes = list(map(list, set_bboxes))
-
-    output_image = deepcopy(image)
-    draw = ImageDraw.Draw(output_image)
-    #TODO: change from local to online path
-    font = ImageFont.truetype("./DejaVuSans.ttf", 28)
-    for i in range(len(list_bboxes)):
-        x1, y1, x2, y2 = list_old_bboxes[i]
-        x1_new, y1_new, x2_new, y2_new = list_bboxes[i]
-        obj_string = '[obj{}]'.format(i)
-        for round_i in text:
-            model_output = round_i[1]
-            model_output = model_output.replace('[{}, {}, {}, {}]'.format(x1, y1, x2, y2), obj_string)
-            round_i[1] = model_output
-        draw.rectangle([(x1_new, y1_new), (x2_new, y2_new)], outline="red", width=3)
-        draw.text((x1_new+2, y1_new+5), obj_string[1:-1], fill="red", font=font)
-
-    return (output_image, [chatbot[0]] + text, disable_btn)
-
-
-def add_text(state, text, image_process_mode, original_image, sketch_pad, request: gr.Request):
-    print("add text called!")
-
-    
-    image = sketch_pad['image']
-    print("text", text, "and  : ", len(text))
-    print("Image path", original_image)
-
-    if len(text) <= 0 and image is None:
-        state.skip_next = True
-        return (state, state.to_gradio_chatbot(), "", None) + (no_change_btn,) * 5
-    if args.moderate:
-        flagged = violates_moderation(text)
-        if flagged:
-            state.skip_next = True
-            return (state, state.to_gradio_chatbot(), moderation_msg, None) + (
-                no_change_btn,) * 5
-
-    text = text[:1536]  # Hard cut-off
-    if original_image is None:
-        assert image is not None
-        original_image = image.copy()
-        print('No location, copy original image in add_text')
-
-    if image is not None:
-        if state.first_round:
-            text = text[:1200]  # Hard cut-off for images
-            if '<image>' not in text:
-                # text = '<Image><image></Image>' + text
-                text = text + '\n<image>'
-            text = (text, original_image, image_process_mode)
-            if len(state.get_images(return_pil=True)) > 0:
-                new_state = default_conversation.copy()
-                new_state.first_round = False
-                state=new_state
-                print('First round add image finsihed.')
-
-    state.append_message(state.roles[0], text)
-    state.append_message(state.roles[1], None)
-    state.skip_next = False
-    return (state, state.to_gradio_chatbot(), "", original_image) + (disable_btn,) * 5
-
-
-def post_process_code(code):
-    sep = "\n```"
-    if sep in code:
-        blocks = code.split(sep)
-        if len(blocks) % 2 == 1:
-            for i in range(1, len(blocks), 2):
-                blocks[i] = blocks[i].replace("\\_", "_")
-        code = sep.join(blocks)
-    return code
-
-
-def find_indices_in_order(str_list, STR):
-    indices = []
-    i = 0
-    while i < len(STR):
-        for element in str_list:
-            if STR[i:i+len(element)] == element:
-                indices.append(str_list.index(element))
-                i += len(element) - 1
-                break
-        i += 1
-    return indices
-
-
-def format_region_prompt(prompt, refer_input_state):
-    # Find regions in prompts and assign corresponding region masks
-    refer_input_state['region_masks_in_prompts'] = []
-    indices_region_placeholder_in_prompt = find_indices_in_order(refer_input_state['region_placeholder_tokens'], prompt)
-    refer_input_state['region_masks_in_prompts'] = [refer_input_state['region_masks'][iii] for iii in indices_region_placeholder_in_prompt]
-
-    # Find regions in prompts and replace with real coordinates and region feature token.
-    for region_ph_index, region_ph_i in enumerate(refer_input_state['region_placeholder_tokens']):
-        prompt = prompt.replace(region_ph_i, '{} {}'.format(refer_input_state['region_coordinates'][region_ph_index], DEFAULT_REGION_FEA_TOKEN))
-    return prompt
-    
-@spaces.GPU()
-def http_bot(state, model_selector, temperature, top_p, max_new_tokens, refer_input_state, request: gr.Request):
-# def http_bot(state, model_selector, temperature, top_p, max_new_tokens, request: gr.Request):
-    start_tstamp = time.time()
-    model_name = model_selector
-
-    if state.skip_next:
-        # This generate call is skipped due to invalid inputs
-        yield (state, state.to_gradio_chatbot()) + (no_change_btn,) * 5
-        return
-
-    print("state messages: ", state.messages)
-    if len(state.messages) == state.offset + 2:
-        # First round of conversation
-        # template_name = 'ferret_v1'
-        template_name = 'ferret_gemma_instruct'
-        # Below is LLaVA's original templates.
-        # if "llava" in model_name.lower():
-        #     if 'llama-2' in model_name.lower():
-        #         template_name = "llava_llama_2"
-        #     elif "v1" in model_name.lower():
-        #         if 'mmtag' in model_name.lower():
-        #             template_name = "v1_mmtag"
-        #         elif 'plain' in model_name.lower() and 'finetune' not in model_name.lower():
-        #             template_name = "v1_mmtag"
-        #         else:
-        #             template_name = "llava_v1"
-        #     elif "mpt" in model_name.lower():
-        #         template_name = "mpt"
-        #     else:
-        #         if 'mmtag' in model_name.lower():
-        #             template_name = "v0_mmtag"
-        #         elif 'plain' in model_name.lower() and 'finetune' not in model_name.lower():
-        #             template_name = "v0_mmtag"
-        #         else:
-        #             template_name = "llava_v0"
-        # elif "mpt" in model_name:
-        #     template_name = "mpt_text"
-        # elif "llama-2" in model_name:
-        #     template_name = "llama_2"
-        # else:
-        #     template_name = "vicuna_v1"
-        new_state = conv_templates[template_name].copy()
-        new_state.append_message(new_state.roles[0], state.messages[-2][1])
-        new_state.append_message(new_state.roles[1], None)
-        state = new_state
-        state.first_round = False
-
-    # # Query worker address
-    # controller_url = args.controller_url
-    # ret = requests.post(controller_url + "/get_worker_address",
-    #         json={"model": model_name})
-    # worker_addr = ret.json()["address"]
-    # logger.info(f"model_name: {model_name}, worker_addr: {worker_addr}")
-
-    # No available worker
-    # if worker_addr == "":
-    #     state.messages[-1][-1] = server_error_msg
-    #     yield (state, state.to_gradio_chatbot(), disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
-    #     return
-
-    # Construct prompt
-    prompt = state.get_prompt()
-    if args.add_region_feature:
-        prompt = format_region_prompt(prompt, refer_input_state)
-
-    all_images = state.get_images(return_pil=True)
-    all_image_hash = [hashlib.md5(image.tobytes()).hexdigest() for image in all_images]
-    for image, hash in zip(all_images, all_image_hash):
-        t = datetime.datetime.now()
-        # fishy can remove it
-        filename = os.path.join(LOGDIR, "serve_images", f"{t.year}-{t.month:02d}-{t.day:02d}", f"{hash}.jpg")
-        if not os.path.isfile(filename):
-            os.makedirs(os.path.dirname(filename), exist_ok=True)
-            image.save(filename)
-
-    # Make requests
-    pload = {
-        "model": model_name,
-        "prompt": prompt,
-        "temperature": float(temperature),
-        "top_p": float(top_p),
-        "max_new_tokens": min(int(max_new_tokens), 1536),
-        "stop": state.sep if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT] else state.sep2,
-        "images": f'List of {len(state.get_images())} images: {all_image_hash}',
-    }
-    logger.info(f"==== request ====\n{pload}")
-    if args.add_region_feature:
-        pload['region_masks'] = refer_input_state['region_masks_in_prompts']
-        logger.info(f"==== add region_masks_in_prompts to request ====\n")
-
-    pload['images'] = state.get_images()
-    print(f'Input Prompt: {prompt}')
-    print("all_image_hash", all_image_hash)
-
-    state.messages[-1][-1] = "▌"
-    yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
-    
-    try:
-        # Stream output
-        stop = state.sep if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT] else state.sep2
-        #TODO: define inference and run function
-        results, extracted_texts = inference_and_run(
-        image_path=all_image_hash[0], # double check this
-        prompt=prompt,
-        model_path=model_name,
-        conv_mode="ferret_gemma_instruct",  # Default mode from the original function
-        temperature=temperature, 
-        top_p=top_p,
-        max_new_tokens=max_new_tokens,
-        stop=stop    # Assuming we want to process the image
-        )
-
-        # response = requests.post(worker_addr + "/worker_generate_stream",
-        #     headers=headers, json=pload, stream=True, timeout=10)
-        response = extracted_texts
-        logger.info(f"This is the respone {response}")
-
-        for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"):
-            if chunk:
-                data = json.loads(chunk.decode())
-                if data["error_code"] == 0:
-                    output = data["text"][len(prompt):].strip()
-                    output = post_process_code(output)
-                    state.messages[-1][-1] = output + "▌"
-                    yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5
-                else:
-                    output = data["text"] + f" (error_code: {data['error_code']})"
-                    state.messages[-1][-1] = output
-                    yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
-                    return
-                time.sleep(0.03)
-    except requests.exceptions.RequestException as e:
-        state.messages[-1][-1] = server_error_msg
-        yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn)
-        return
-
-    state.messages[-1][-1] = state.messages[-1][-1][:-1]
-    yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5
-
-    finish_tstamp = time.time()
-    logger.info(f"{output}")
-
-    with open(get_conv_log_filename(), "a") as fout:
-        data = {
-            "tstamp": round(finish_tstamp, 4),
-            "type": "chat",
-            "model": model_name,
-            "start": round(start_tstamp, 4),
-            "finish": round(start_tstamp, 4),
-            "state": state.dict(),
-            "images": all_image_hash,
-            "ip": request.client.host,
-        }
-        fout.write(json.dumps(data) + "\n")
-
-title_markdown = ("""
-# 🦦 Ferret: Refer and Ground Anything Anywhere at Any Granularity
-""")
-# [[Project Page]](https://llava-vl.github.io) [[Paper]](https://arxiv.org/abs/2304.08485)
-
-tos_markdown = ("""
-### Terms of use
-By using this service, users are required to agree to the following terms: The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. The service may collect user dialogue data for future research.
-""")
-
-
-learn_more_markdown = ("""
-### License
-The service is a research preview intended for non-commercial use only
-""")
-
-
-css = code_highlight_css + """
-pre {
-    white-space: pre-wrap;       /* Since CSS 2.1 */
-    white-space: -moz-pre-wrap;  /* Mozilla, since 1999 */
-    white-space: -pre-wrap;      /* Opera 4-6 */
-    white-space: -o-pre-wrap;    /* Opera 7 */
-    word-wrap: break-word;       /* Internet Explorer 5.5+ */
-}
-"""
-
-Instructions = '''
-Instructions:
-1. Select a 'Referring Input Type'
-2. Draw on the image to refer to a region/point.
-3. Copy the region id from 'Referring Input Type' to refer to a region in your chat.
-'''
-from gradio.events import Dependency
-
-class ImageMask(gr.components.Image):
-    """
-    Sets: source="canvas", tool="sketch"
-    """
-
-    is_template = True
-
-    def __init__(self, **kwargs):
-        super().__init__(source="upload", tool="sketch", interactive=True, **kwargs)
-
-    def preprocess(self, x):
-        return super().preprocess(x)
-    from typing import Callable, Literal, Sequence, Any, TYPE_CHECKING
-    from gradio.blocks import Block
-    if TYPE_CHECKING:
-        from gradio.components import Timer
-    
-
-def draw(input_mode, input, refer_input_state, refer_text_show, imagebox_refer):
-    if type(input) == dict:
-        image = deepcopy(input['image'])
-        mask = deepcopy(input['mask'])
-    else:
-        mask = deepcopy(input)
-
-    # W, H -> H, W, 3
-    image_new = np.asarray(image)
-    img_height = image_new.shape[0]
-    img_width = image_new.shape[1]
-
-    # W, H, 4 -> H, W
-    mask_new = np.asarray(mask)[:,:,0].copy()
-    mask_new = torch.from_numpy(mask_new)
-    mask_new = (F.interpolate(mask_new.unsqueeze(0).unsqueeze(0), (img_height, img_width), mode='bilinear') > 0)
-    mask_new = mask_new[0, 0].transpose(1, 0).long()
-
-    if len(refer_input_state['masks']) == 0:
-        last_mask = torch.zeros_like(mask_new)
-    else:
-        last_mask = refer_input_state['masks'][-1]
-
-    diff_mask = mask_new - last_mask
-    if torch.all(diff_mask == 0):
-        print('Init Uploading Images.')
-        return (refer_input_state, refer_text_show, image)
-    else:
-        refer_input_state['masks'].append(mask_new)
-
-    if input_mode == 'Point':
-        nonzero_points = diff_mask.nonzero()
-        nonzero_points_avg_x = torch.median(nonzero_points[:, 0])
-        nonzero_points_avg_y = torch.median(nonzero_points[:, 1])
-        sampled_coor = [nonzero_points_avg_x, nonzero_points_avg_y]
-        # pdb.set_trace()
-        cur_region_masks = generate_mask_for_feature(sampled_coor, raw_w=img_width, raw_h=img_height)
-    elif input_mode == 'Box' or input_mode == 'Sketch':
-        # pdb.set_trace()
-        x1x2 = diff_mask.max(1)[0].nonzero()[:, 0]
-        y1y2 = diff_mask.max(0)[0].nonzero()[:, 0]
-        y1, y2 = y1y2.min(), y1y2.max()
-        x1, x2 = x1x2.min(), x1x2.max()
-        # pdb.set_trace()
-        sampled_coor = [x1, y1, x2, y2]
-        if input_mode == 'Box':
-            cur_region_masks = generate_mask_for_feature(sampled_coor, raw_w=img_width, raw_h=img_height)
-        else:
-            cur_region_masks = generate_mask_for_feature(sampled_coor, raw_w=img_width, raw_h=img_height, mask=diff_mask)
-    else:
-        raise NotImplementedError(f'Input mode of {input_mode} is not Implemented.')
-
-    # TODO(haoxuan): Hack img_size to be 224 here, need to make it a argument.
-    if len(sampled_coor) == 2:
-        point_x = int(VOCAB_IMAGE_W * sampled_coor[0] / img_width)
-        point_y = int(VOCAB_IMAGE_H * sampled_coor[1] / img_height)
-        cur_region_coordinates = f'[{int(point_x)}, {int(point_y)}]'
-    elif len(sampled_coor) == 4:
-        point_x1 = int(VOCAB_IMAGE_W * sampled_coor[0] / img_width)
-        point_y1 = int(VOCAB_IMAGE_H * sampled_coor[1] / img_height)
-        point_x2 = int(VOCAB_IMAGE_W * sampled_coor[2] / img_width)
-        point_y2 = int(VOCAB_IMAGE_H * sampled_coor[3] / img_height)
-        cur_region_coordinates = f'[{int(point_x1)}, {int(point_y1)}, {int(point_x2)}, {int(point_y2)}]'
-
-    cur_region_id = len(refer_input_state['region_placeholder_tokens'])
-    cur_region_token = DEFAULT_REGION_REFER_TOKEN.split(']')[0] + str(cur_region_id) + ']'
-    refer_input_state['region_placeholder_tokens'].append(cur_region_token)
-    refer_input_state['region_coordinates'].append(cur_region_coordinates)
-    refer_input_state['region_masks'].append(cur_region_masks)
-    assert len(refer_input_state['region_masks']) == len(refer_input_state['region_coordinates']) == len(refer_input_state['region_placeholder_tokens'])
-    refer_text_show.append((cur_region_token, ''))
-
-    # Show Parsed Referring.
-    imagebox_refer = draw_box(sampled_coor, cur_region_masks, \
-                         cur_region_token, imagebox_refer, input_mode)
-
-    return (refer_input_state, refer_text_show, imagebox_refer)
-
-def build_demo(embed_mode):
-    textbox = gr.Textbox(show_label=False, placeholder="Enter text and press ENTER", visible=False, container=False)
-    with gr.Blocks(title="FERRET", theme=gr.themes.Base(), css=css) as demo:
-        state = gr.State()
-
-        if not embed_mode:
-            gr.Markdown(title_markdown)
-            gr.Markdown(Instructions)
-
-        with gr.Row():
-            with gr.Column(scale=4):
-                with gr.Row(elem_id="model_selector_row"):
-                    model_selector = gr.Dropdown(
-                        choices=models,
-                        value=models[0] if len(models) > 0 else "",
-                        interactive=True,
-                        show_label=False,
-                        container=False)
-
-                original_image = gr.Image(type="pil", visible=False)
-                image_process_mode = gr.Radio(
-                    ["Raw+Processor", "Crop", "Resize", "Pad"],
-                    value="Raw+Processor",
-                    label="Preprocess for non-square image",
-                    visible=False)
-
-                # Added for any-format input.
-                sketch_pad = ImageMask(label="Image & Sketch", type="pil", elem_id="img2text")
-                refer_input_mode = gr.Radio(
-                    ["Point", "Box", "Sketch"],
-                    value="Point",
-                    label="Referring Input Type")
-                refer_input_state = gr.State({'region_placeholder_tokens':[],
-                                              'region_coordinates':[],
-                                              'region_masks':[],
-                                              'region_masks_in_prompts':[],
-                                              'masks':[],
-                                              })
-                refer_text_show = gr.HighlightedText(value=[], label="Referring Input Cache")
-
-                imagebox_refer = gr.Image(type="pil", label="Parsed Referring Input")
-                imagebox_output = gr.Image(type="pil", label='Output Vis')
-
-                cur_dir = os.path.dirname(os.path.abspath(__file__))
-                # gr.Examples(examples=[
-                #     # [f"{cur_dir}/examples/harry-potter-hogwarts.jpg", "What is in [region0]? And what do people use it for?"],
-                #     # [f"{cur_dir}/examples/ingredients.jpg", "What objects are in [region0] and [region1]?"],
-                #     # [f"{cur_dir}/examples/extreme_ironing.jpg", "What is unusual about this image? And tell me the coordinates of mentioned objects."],
-                #     [f"{cur_dir}/examples/ferret.jpg", "What's the relationship between object [region0] and object [region1]?"],
-                #     [f"{cur_dir}/examples/waterview.jpg", "What are the things I should be cautious about when I visit here? Tell me the coordinates in response."],
-                #     [f"{cur_dir}/examples/flickr_9472793441.jpg", "Describe the image in details."],
-                #     # [f"{cur_dir}/examples/coco_000000281759.jpg", "What are the locations of the woman wearing a blue dress, the woman in flowery top, the girl in purple dress, the girl wearing green shirt?"],
-                #     [f"{cur_dir}/examples/room_planning.jpg", "How to improve the design of the given room?"],
-                #     [f"{cur_dir}/examples/make_sandwitch.jpg", "How can I make a sandwich with available ingredients?"],
-                #     [f"{cur_dir}/examples/bathroom.jpg", "What is unusual about this image?"],
-                #     [f"{cur_dir}/examples/kitchen.png", "Is the object a man or a chicken? Explain the reason."],
-                # ], inputs=[sketch_pad, textbox])
-
-                with gr.Accordion("Parameters", open=False, visible=False) as parameter_row:
-                    temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.2, step=0.1, interactive=True, label="Temperature",)
-                    top_p = gr.Slider(minimum=0.0, maximum=1.0, value=0.7, step=0.1, interactive=True, label="Top P",)
-                    max_output_tokens = gr.Slider(minimum=0, maximum=1024, value=512, step=64, interactive=True, label="Max output tokens",)
-
-            with gr.Column(scale=5):
-                chatbot = gr.Chatbot(elem_id="chatbot", label="FERRET", visible=False).style(height=750)
-                with gr.Row():
-                    with gr.Column(scale=8):
-                        textbox.render()
-                    with gr.Column(scale=1, min_width=60):
-                        submit_btn = gr.Button(value="Submit", visible=False)
-                with gr.Row(visible=False) as button_row:
-                    upvote_btn = gr.Button(value="👍  Upvote", interactive=False)
-                    downvote_btn = gr.Button(value="👎  Downvote", interactive=False)
-                    # flag_btn = gr.Button(value="⚠️  Flag", interactive=False)
-                    #stop_btn = gr.Button(value="⏹️  Stop Generation", interactive=False)
-                    regenerate_btn = gr.Button(value="🔄  Regenerate", interactive=False)
-                    clear_btn = gr.Button(value="🗑️  Clear history", interactive=False)
-                    location_btn = gr.Button(value="🪄 Show location", interactive=False)
-
-        if not embed_mode:
-            gr.Markdown(tos_markdown)
-            gr.Markdown(learn_more_markdown)
-        url_params = gr.JSON(visible=False)
-
-        # Register listeners
-        btn_list = [upvote_btn, downvote_btn, location_btn, regenerate_btn, clear_btn]
-        upvote_btn.click(upvote_last_response,
-            [state, model_selector], [textbox, upvote_btn, downvote_btn, location_btn])
-        downvote_btn.click(downvote_last_response,
-            [state, model_selector], [textbox, upvote_btn, downvote_btn, location_btn])
-        # flag_btn.click(flag_last_response,
-        #     [state, model_selector], [textbox, upvote_btn, downvote_btn, flag_btn])
-        regenerate_btn.click(regenerate, [state, image_process_mode],
-            [state, chatbot, textbox] + btn_list).then(
-            http_bot, [state, model_selector, temperature, top_p, max_output_tokens, refer_input_state],
-            [state, chatbot] + btn_list)
-        clear_btn.click(clear_history, None, [state, chatbot, textbox, imagebox_output, original_image] + btn_list + \
-                        [sketch_pad, refer_input_state, refer_text_show, imagebox_refer])
-        location_btn.click(show_location,
-            [sketch_pad, chatbot], [imagebox_output, chatbot, location_btn])
-
-
-        #TODO: fix bug text and image not adding when clicking submit
-        textbox.submit(add_text, [state, textbox, image_process_mode, original_image, sketch_pad], [state, chatbot, textbox, original_image] + btn_list
-            ).then(http_bot, [state, model_selector, temperature, top_p, max_output_tokens, refer_input_state],
-                   [state, chatbot] + btn_list)
-
-        submit_btn.click(add_text, [state, textbox, image_process_mode, original_image, sketch_pad], [state, chatbot, textbox, original_image] + btn_list
-            ).then(http_bot, [state, model_selector, temperature, top_p, max_output_tokens, refer_input_state],
-                   [state, chatbot] + btn_list)
-
-            
-
-        sketch_pad.edit(
-            draw,
-            inputs=[refer_input_mode, sketch_pad, refer_input_state, refer_text_show, imagebox_refer],
-            outputs=[refer_input_state, refer_text_show, imagebox_refer],
-            queue=True,
-        )
-
-        if args.model_list_mode == "once":
-            demo.load(load_demo, [url_params], [state, model_selector,
-                chatbot, textbox, submit_btn, button_row, parameter_row],
-                _js=get_window_url_params)
-        elif args.model_list_mode == "reload":
-            demo.load(load_demo_refresh_model_list, None, [state, model_selector,
-                chatbot, textbox, submit_btn, button_row, parameter_row])
-        else:
-            raise ValueError(f"Unknown model list mode: {args.model_list_mode}")
-
-    return demo
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--host", type=str, default="0.0.0.0")
-    parser.add_argument("--port", type=int)
-    parser.add_argument("--controller-url", type=str, default="http://localhost:21001")
-    parser.add_argument("--concurrency-count", type=int, default=8)
-    parser.add_argument("--model-list-mode", type=str, default="once",
-        choices=["once", "reload"])
-    parser.add_argument("--share", action="store_true")
-    parser.add_argument("--moderate", action="store_true")
-    parser.add_argument("--embed", action="store_true")
-    parser.add_argument("--add_region_feature", action="store_true")
-    args = parser.parse_args()
-    logger.info(f"args: {args}")
-
-    models = get_model_list()
-
-    logger.info(args)
-    demo = build_demo(args.embed)
-    demo.queue(concurrency_count=args.concurrency_count, status_update_rate=10,
-               api_open=False).launch(
-        server_name=args.host, server_port=args.port, share=True)

cli.py

@@ -1,130 +1,164 @@
-import argparse
 import torch
-
 from constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
 from conversation import conv_templates, SeparatorStyle
 from builder import load_pretrained_model
 from utils import disable_torch_init
-from ferretui.mm_utils import process_images, tokenizer_image_token, get_model_name_from_path
-
+from mm_utils import process_images, tokenizer_image_token, get_model_name_from_path
 from PIL import Image
-
 import requests
-from PIL import Image
 from io import BytesIO
 from transformers import TextStreamer
-
+import spaces
+from functools import partial
+import traceback
+import sys
+# def load_image(image_file):
+#     if image_file.startswith('http://') or image_file.startswith('https://'):
+#         response = requests.get(image_file)
+#         image = Image.open(BytesIO(response.content)).convert('RGB')
+#     else:
+#         image = Image.open(image_file).convert('RGB')
+#     return image
 
 def load_image(image_file):
-    if image_file.startswith('http://') or image_file.startswith('https://'):
-        response = requests.get(image_file)
-        image = Image.open(BytesIO(response.content)).convert('RGB')
-    else:
-        image = Image.open(image_file).convert('RGB')
+    print("the image file : ", image_file)
+    
+    image = Image.open(image_file).convert('RGB')
+
+    if image is None:
+        print("image is None")
+        sys.exit("Aborting program: Image is None.")
+    
     return image
 
-
-def main(args):
-    # Model
+@spaces.GPU()
+def run_inference(
+    model_path,
+    image_file,
+    prompt_text,
+    model_base=None,
+    device="cuda",
+    conv_mode=None,
+    temperature=0.2,
+    max_new_tokens=512,
+    load_8bit=False,
+    load_4bit=False,
+    debug=False
+):
+    # Model initialization
     disable_torch_init()
 
-    model_name = get_model_name_from_path(args.model_path)
-    tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, args.load_8bit, args.load_4bit, device=args.device)
+    model_name = get_model_name_from_path(model_path)
+    tokenizer, model, image_processor, context_len = load_pretrained_model(
+        model_path, model_base, model_name, load_8bit, load_4bit
+    )
 
+    # Determine conversation mode
     if "llama-2" in model_name.lower():
-        conv_mode = "llava_llama_2"
+        conv_mode_inferred = "llava_llama_2"
     elif "mistral" in model_name.lower():
-        conv_mode = "mistral_instruct"
+        conv_mode_inferred = "mistral_instruct"
     elif "v1.6-34b" in model_name.lower():
-        conv_mode = "chatml_direct"
+        conv_mode_inferred = "chatml_direct"
     elif "v1" in model_name.lower():
-        conv_mode = "llava_v1"
+        conv_mode_inferred = "llava_v1"
     elif "mpt" in model_name.lower():
-        conv_mode = "mpt"
-    if "gemma" in model_name.lower():
-        conv_mode = "ferret_gemma_instruct"
-    if "llama" in model_name.lower():
-        conv_mode = "ferret_llama_3"
+        conv_mode_inferred = "mpt"
+    elif "gemma" in model_name.lower():
+        conv_mode_inferred = "ferret_gemma_instruct"
+    elif "llama" in model_name.lower():
+        conv_mode_inferred = "ferret_llama_3"
     else:
-        conv_mode = "llava_v0"
+        conv_mode_inferred = "llava_v0"
 
-    if args.conv_mode is not None and conv_mode != args.conv_mode:
-        print('[WARNING] the auto inferred conversation mode is {}, while `--conv-mode` is {}, using {}'.format(conv_mode, args.conv_mode, args.conv_mode))
-    else:
-        args.conv_mode = conv_mode
+    # Use user-specified conversation mode if provided
+    conv_mode = conv_mode or conv_mode_inferred
+
+    if conv_mode != conv_mode_inferred:
+        print(f'[WARNING] the auto inferred conversation mode is {conv_mode_inferred}, while `conv_mode` is {conv_mode}, using {conv_mode}')
+
+    conv = conv_templates[conv_mode].copy()
 
-    conv = conv_templates[args.conv_mode].copy()
     if "mpt" in model_name.lower():
         roles = ('user', 'assistant')
     else:
         roles = conv.roles
 
-    image = load_image(args.image_file)
+    # Load and process image
+    print("loading image", image_file)
+    image = load_image(image_file)
+    if image is None:
+        print("image is None")
     image_size = image.size
-    # Similar operation in model_worker.py
-    image_tensor = process_images([image], image_processor, model.config)
-    if type(image_tensor) is list:
-        image_tensor = [image.to(model.device, dtype=torch.float16) for image in image_tensor]
+    image_h = 336  # Height of the image
+    image_w = 336
+    #ERROR
+    # image_tensor = process_images([image], image_processor, model.config)
+    # if type(image_tensor) is list:
+    #     image_tensor = [image.to(model.device, dtype=torch.float16) for image in image_tensor]
+    # else:
+    #     image_tensor = image_tensor.to(model.device, dtype=torch.float16)
+    if model.config.image_aspect_ratio == "square_nocrop":
+            image_tensor = image_processor.preprocess(image, return_tensors='pt', do_resize=True, 
+                                                  do_center_crop=False, size=[image_h, image_w])['pixel_values'][0]
+    elif model.config.image_aspect_ratio == "anyres":
+        image_process_func = partial(image_processor.preprocess, return_tensors='pt', do_resize=True, do_center_crop=False, size=[image_h, image_w])
+        image_tensor = process_images([image], image_processor, model.config, image_process_func=image_process_func)[0]
+    else:
+        image_tensor = process_images([image], image_processor, model.config)[0]
+
+    if model.dtype == torch.float16:
+        image_tensor = image_tensor.half()  # Convert image tensor to float16
+        data_type = torch.float16
+    else:
+        image_tensor = image_tensor.float()  # Keep it in float32
+        data_type = torch.float32
+
+    # Now, add the batch dimension and move to GPU
+    images = image_tensor.unsqueeze(0).to(data_type).cuda()
+
+
+    # Process the first message with the image
+    if model.config.mm_use_im_start_end:
+        prompt_text = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + prompt_text
     else:
-        image_tensor = image_tensor.to(model.device, dtype=torch.float16)
-
-    while True:
-        try:
-            inp = input(f"{roles[0]}: ")
-        except EOFError:
-            inp = ""
-        if not inp:
-            print("exit...")
-            break
-
-        print(f"{roles[1]}: ", end="")
-
-        if image is not None:
-            # first message
-            if model.config.mm_use_im_start_end:
-                inp = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + inp
-            else:
-                inp = DEFAULT_IMAGE_TOKEN + '\n' + inp
-            image = None
-        
-        conv.append_message(conv.roles[0], inp)
-        conv.append_message(conv.roles[1], None)
-        prompt = conv.get_prompt()
-
-        input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(model.device)
-        stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
-        keywords = [stop_str]
-        streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
-
-        with torch.inference_mode():
-            output_ids = model.generate(
-                input_ids,
-                images=image_tensor,
-                image_sizes=[image_size],
-                do_sample=True if args.temperature > 0 else False,
-                temperature=args.temperature,
-                max_new_tokens=args.max_new_tokens,
-                streamer=streamer,
-                use_cache=True)
-
-        outputs = tokenizer.decode(output_ids[0]).strip()
-        conv.messages[-1][-1] = outputs
-
-        if args.debug:
-            print("\n", {"prompt": prompt, "outputs": outputs}, "\n")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--model-path", type=str, default="facebook/opt-350m")
-    parser.add_argument("--model-base", type=str, default=None)
-    parser.add_argument("--image-file", type=str, required=True)
-    parser.add_argument("--device", type=str, default="cuda")
-    parser.add_argument("--conv-mode", type=str, default=None)
-    parser.add_argument("--temperature", type=float, default=0.2)
-    parser.add_argument("--max-new-tokens", type=int, default=512)
-    parser.add_argument("--load-8bit", action="store_true")
-    parser.add_argument("--load-4bit", action="store_true")
-    parser.add_argument("--debug", action="store_true")
-    args = parser.parse_args()
-    main(args)
+        prompt_text = DEFAULT_IMAGE_TOKEN + '\n' + prompt_text
+
+    # Prepare conversation
+    conv.append_message(conv.roles[0], prompt_text)
+    conv.append_message(conv.roles[1], None)
+    prompt = conv.get_prompt()
+
+    input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(model.device)
+    stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+    keywords = [stop_str]
+    streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+    print("image size: ", image_size)
+    # Generate the model's response
+    
+    with torch.inference_mode():
+        output_ids = model.generate(
+            input_ids,
+            images=images,
+            image_sizes=[image_size],
+            do_sample=True if temperature > 0 else False,
+            temperature=temperature,
+            max_new_tokens=max_new_tokens,
+            streamer=streamer,
+            num_beams=1,
+            use_cache=True
+        )
+
+    # Decode and return the output
+    outputs = tokenizer.decode(output_ids[0]).strip()
+    conv.messages[-1][-1] = outputs
+
+    if debug:
+        print("\n", {"prompt": prompt, "outputs": outputs}, "\n")
+
+    return outputs
+
+
+# Example usage:
+# response = run_inference("path_to_model", "path_to_image", "your_prompt")
+# print(response)

eval.json

@@ -1 +1 @@
-[{"id": 0, "image": "8b23f327b90b6211049acd36e3f99975.jpg", "image_h": 433, "image_w": 400, "conversations": [{"from": "human", "value": "<image>\nA chat between a human and an AI that understands visuals. In images, [x, y] denotes points: top-left [0, 0], bottom-right [width-1, height-1]. Increasing x moves right; y moves down. Bounding box: [x1, y1, x2, y2]. Image size: 1000x1000. Follow instructions.<start_of_turn>user\n<image>\nexplain what you see<end_of_turn>\n<start_of_turn>model\n"}]}]
+[{"id": 0, "image": "Screenshot 2024-10-13 at 12.01.05\u202fAM.png", "image_h": 76, "image_w": 90, "conversations": [{"from": "human", "value": "<image>\ndescribe what you see in details"}]}]

eval_output.jsonl/0_of_1.jsonl

@@ -1 +1 @@
-{"id": 0, "image_path": "8b23f327b90b6211049acd36e3f99975.jpg", "prompt": "A chat between a human and an AI that understands visuals. In images, [x, y] denotes points: top-left [0, 0], bottom-right [width-1, height-1]. Increasing x moves right; y moves down. Bounding box: [x1, y1, x2, y2]. Image size: 1000x1000. Follow instructions.<start_of_turn>user", "text": "A chat between a human and an unknown entity. \n\nThe conversation starts with a message from Jackyline Herrera saying, \"Ask Jackie to borrow her truck\". The reply is, \"Get gravel for bow, walk, 10, 1, 1, Shopping List\". \n\nThe next message is from Get Gravel for the truck, and the reply is, \"Buy mulch, #shoppinglist\". \n\nThe third message is from Buy mulch for the garden, and the reply is, \"Pick up succulents\". \n\nThe fourth message is from Pick up succulents for the garden, and the reply is, \"Buy soil for succulents\". \n\nThe fifth message is from Buy soil for succulents, and the reply is, \"Pick up soil for succulents\". \n\nThe sixth message is from Pick up succulents for the garden, and the reply is, \"Pick up soil for succulents\". \n\nThe seventh message is from Pick up succulents for the garden, and the reply is, \"Pick up soil for succulents\". \n\nThe eighth message is from Pick up succulents for the garden, and the reply is, \"Pick up soil for succulents\". \n\nThe ninth message is from Pick up succulents for the garden, and the reply is, \"Look up native vegetables along the fence\". \n\nThe tenth message is from Shopping List, and the reply is, \"Shopping List\". \n\nThe message at the bottom is from Shopping List, and the reply is, \"Look up native vegetables along the fence\". \n\nThe message at the very bottom is from Shopping List, and the reply is, \"Looking: Fran\".", "label": null}
+{"id": 0, "image_path": "Screenshot 2024-10-13 at 12.01.05\u202fAM.png", "prompt": "describe what you see in details", "text": "The screen contains a large picture that occupies most of the screen, extending from nearly the top to the bottom. In the lower portion of the screen, there is a button labeled \"menu\". The button is relatively large and positioned at the lower part of the screen.", "label": null}