Upload app(1).py

app(1).py

@@ -0,0 +1,375 @@
+import argparse
+import hashlib
+import json
+import os
+import time
+from threading import Thread
+
+import gradio as gr
+import torch
+from llava.constants import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN,
+ DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX)
+from llava.conversation import (SeparatorStyle, conv_templates,
+ default_conversation)
+from llava.mm_utils import (KeywordsStoppingCriteria, load_image_from_base64,
+ process_images, tokenizer_image_token)
+from llava.model.builder import load_pretrained_model
+from transformers import TextIteratorStreamer
+
+print(gr.__version__)
+
+block_css = """
+
+#buttons button {
+ min-width: min(120px,100%);
+}
+"""
+title_markdown = ("""
+# 🐬 ShareGPT4V: Improving Large Multi-modal Models with Better Captions
+### 🔊 Notice: The demo of Share-Captioner will soon be supported. Stay tune for updates!
+[[Project Page](https://sharegpt4v.github.io/)] [[Code](https://github.com/InternLM/InternLM-XComposer/tree/main/projects/ShareGPT4V)] | 📚 [[Paper](https://arxiv.org/pdf/2311.12793.pdf)]
+""")
+tos_markdown = ("""
+### Terms of use
+By using this service, users are required to agree to the following terms:
+The service is a research preview intended for non-commercial use only. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes.
+For an optimal experience, please use desktop computers for this demo, as mobile devices may compromise its quality.
+""")
+learn_more_markdown = ("""
+### License
+The service is a research preview intended for non-commercial use only, subject to the model [License](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) of LLaMA, [Terms of Use](https://openai.com/policies/terms-of-use) of the data generated by OpenAI, and [Privacy Practices](https://chrome.google.com/webstore/detail/sharegpt-share-your-chatg/daiacboceoaocpibfodeljbdfacokfjb) of ShareGPT. Please contact us if you find any potential violation.
+""")
+ack_markdown = ("""
+### Acknowledgement
+The template for this web demo is from [LLaVA](https://github.com/haotian-liu/LLaVA), and we are very grateful to LLaVA for their open source contributions to the community!
+""")
+
+
+def regenerate(state, image_process_mode):
+ state.messages[-1][-1] = None
+ prev_human_msg = state.messages[-2]
+ if type(prev_human_msg[1]) in (tuple, list):
+ prev_human_msg[1] = (*prev_human_msg[1][:2], image_process_mode)
+ state.skip_next = False
+ return (state, state.to_gradio_chatbot(), "", None)
+
+
+def clear_history():
+ state = default_conversation.copy()
+ return (state, state.to_gradio_chatbot(), "", None)
+
+
+def add_text(state, text, image, image_process_mode):
+ if len(text) <= 0 and image is None:
+ state.skip_next = True
+ return (state, state.to_gradio_chatbot(), "", None)
+
+ text = text[:1536] # Hard cut-off
+ if image is not None:
+ text = text[:1200] # Hard cut-off for images
+ if '<image>' not in text:
+ # text = '<Image><image></Image>' + text
+ text = text + '\n<image>'
+ text = (text, image, image_process_mode)
+ if len(state.get_images(return_pil=True)) > 0:
+ state = default_conversation.copy()
+ state.append_message(state.roles[0], text)
+ state.append_message(state.roles[1], None)
+ state.skip_next = False
+ return (state, state.to_gradio_chatbot(), "", None)
+
+
+def load_demo():
+ state = default_conversation.copy()
+ return state
+
+
+@torch.inference_mode()
+def get_response(params):
+ prompt = params["prompt"]
+ ori_prompt = prompt
+ images = params.get("images", None)
+ num_image_tokens = 0
+ if images is not None and len(images) > 0:
+ if len(images) > 0:
+ if len(images) != prompt.count(DEFAULT_IMAGE_TOKEN):
+ raise ValueError(
+ "Number of images does not match number of <image> tokens in prompt")
+
+ images = [load_image_from_base64(image) for image in images]
+ images = process_images(images, image_processor, model.config)
+
+ if type(images) is list:
+ images = [image.to(model.device, dtype=torch.float16)
+ for image in images]
+ else:
+ images = images.to(model.device, dtype=torch.float16)
+
+ replace_token = DEFAULT_IMAGE_TOKEN
+ if getattr(model.config, 'mm_use_im_start_end', False):
+ replace_token = DEFAULT_IM_START_TOKEN + replace_token + DEFAULT_IM_END_TOKEN
+ prompt = prompt.replace(DEFAULT_IMAGE_TOKEN, replace_token)
+
+ num_image_tokens = prompt.count(
+ replace_token) * model.get_vision_tower().num_patches
+ else:
+ images = None
+ image_args = {"images": images}
+ else:
+ images = None
+ image_args = {}
+
+ temperature = float(params.get("temperature", 1.0))
+ top_p = float(params.get("top_p", 1.0))
+ max_context_length = getattr(
+ model.config, 'max_position_embeddings', 2048)
+ max_new_tokens = min(int(params.get("max_new_tokens", 256)), 1024)
+ stop_str = params.get("stop", None)
+ do_sample = True if temperature > 0.001 else False
+
+ input_ids = tokenizer_image_token(
+ prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(model.device)
+ keywords = [stop_str]
+ stopping_criteria = KeywordsStoppingCriteria(
+ keywords, tokenizer, input_ids)
+ streamer = TextIteratorStreamer(
+ tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=15)
+
+ max_new_tokens = min(max_new_tokens, max_context_length -
+ input_ids.shape[-1] - num_image_tokens)
+
+ if max_new_tokens < 1:
+ yield json.dumps({"text": ori_prompt + "Exceeds max token length. Please start a new conversation, thanks.", "error_code": 0}).encode() + b"\0"
+ return
+
+ # local inference
+ thread = Thread(target=model.generate, kwargs=dict(
+ inputs=input_ids,
+ do_sample=do_sample,
+ temperature=temperature,
+ top_p=top_p,
+ max_new_tokens=max_new_tokens,
+ streamer=streamer,
+ stopping_criteria=[stopping_criteria],
+ use_cache=True,
+ **image_args
+ ))
+ thread.start()
+
+ generated_text = ori_prompt
+ for new_text in streamer:
+ generated_text += new_text
+ if generated_text.endswith(stop_str):
+ generated_text = generated_text[:-len(stop_str)]
+ yield json.dumps({"text": generated_text, "error_code": 0}).encode()
+
+
+def http_bot(state, temperature, top_p, max_new_tokens):
+ if state.skip_next:
+ # This generate call is skipped due to invalid inputs
+ yield (state, state.to_gradio_chatbot())
+ return
+
+ if len(state.messages) == state.offset + 2:
+ # First round of conversation
+ if "llava" in model_name.lower():
+ if 'llama-2' in model_name.lower():
+ template_name = "llava_llama_2"
+ elif "v1" in model_name.lower():
+ if 'mmtag' in model_name.lower():
+ template_name = "v1_mmtag"
+ elif 'plain' in model_name.lower() and 'finetune' not in model_name.lower():
+ template_name = "v1_mmtag"
+ else:
+ template_name = "llava_v1"
+ elif "mpt" in model_name.lower():
+ template_name = "mpt"
+ else:
+ if 'mmtag' in model_name.lower():
+ template_name = "v0_mmtag"
+ elif 'plain' in model_name.lower() and 'finetune' not in model_name.lower():
+ template_name = "v0_mmtag"
+ else:
+ template_name = "llava_v0"
+ elif "mpt" in model_name:
+ template_name = "mpt_text"
+ elif "llama-2" in model_name:
+ template_name = "llama_2"
+ else:
+ template_name = "vicuna_v1"
+ new_state = conv_templates[template_name].copy()
+ new_state.append_message(new_state.roles[0], state.messages[-2][1])
+ new_state.append_message(new_state.roles[1], None)
+ state = new_state
+
+ # Construct prompt
+ prompt = state.get_prompt()
+
+ all_images = state.get_images(return_pil=True)
+ all_image_hash = [hashlib.md5(image.tobytes()).hexdigest()
+ for image in all_images]
+
+ # Make requests
+ pload = {
+ "model": model_name,
+ "prompt": prompt,
+ "temperature": float(temperature),
+ "top_p": float(top_p),
+ "max_new_tokens": min(int(max_new_tokens), 1536),
+ "stop": state.sep if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT] else state.sep2,
+ "images": f'List of {len(state.get_images())} images: {all_image_hash}',
+ }
+
+ pload['images'] = state.get_images()
+
+ state.messages[-1][-1] = "▌"
+ yield (state, state.to_gradio_chatbot())
+
+ # for stream
+ output = get_response(pload)
+ for chunk in output:
+ if chunk:
+ data = json.loads(chunk.decode())
+ if data["error_code"] == 0:
+ output = data["text"][len(prompt):].strip()
+ state.messages[-1][-1] = output + "▌"
+ yield (state, state.to_gradio_chatbot())
+ else:
+ output = data["text"] + \
+ f" (error_code: {data['error_code']})"
+ state.messages[-1][-1] = output
+ yield (state, state.to_gradio_chatbot())
+ return
+ time.sleep(0.03)
+
+ state.messages[-1][-1] = state.messages[-1][-1][:-1]
+ yield (state, state.to_gradio_chatbot())
+
+
+def build_demo():
+ textbox = gr.Textbox(
+ show_label=False, placeholder="Enter text and press ENTER", container=False)
+ with gr.Blocks(title="ShareGPT4V", theme=gr.themes.Default(), css=block_css) as demo:
+ state = gr.State()
+ gr.Markdown(title_markdown)
+
+ with gr.Row():
+ with gr.Column(scale=5):
+ with gr.Row(elem_id="Model ID"):
+ gr.Dropdown(
+ choices=['ShareGPT4V-7B'],
+ value='ShareGPT4V-7B',
+ interactive=True,
+ label='Model ID',
+ container=False)
+ imagebox = gr.Image(type="pil")
+ image_process_mode = gr.Radio(
+ ["Crop", "Resize", "Pad", "Default"],
+ value="Default",
+ label="Preprocess for non-square image", visible=False)
+
+ cur_dir = os.path.dirname(os.path.abspath(__file__))
+ gr.Examples(examples=[
+ [f"{cur_dir}/examples/breaking_bad.png",
+ "What is the most common catchphrase of the character on the right?"],
+ [f"{cur_dir}/examples/photo.png",
+ "From a photography perspective, analyze what makes this picture beautiful?"],
+ ], inputs=[imagebox, textbox])
+
+ with gr.Accordion("Parameters", open=False) as _:
+ temperature = gr.Slider(
+ minimum=0.0, maximum=1.0, value=0.2, step=0.1, interactive=True, label="Temperature",)
+ top_p = gr.Slider(
+ minimum=0.0, maximum=1.0, value=0.7, step=0.1, interactive=True, label="Top P",)
+ max_output_tokens = gr.Slider(
+ minimum=0, maximum=1024, value=512, step=64, interactive=True, label="Max output tokens",)
+
+ with gr.Column(scale=8):
+ chatbot = gr.Chatbot(
+ elem_id="chatbot", label="ShareGPT4V Chatbot", height=550)
+ with gr.Row():
+ with gr.Column(scale=8):
+ textbox.render()
+ with gr.Column(scale=1, min_width=50):
+ submit_btn = gr.Button(value="Send", variant="primary")
+ with gr.Row(elem_id="buttons") as _:
+ regenerate_btn = gr.Button(
+ value="🔄 Regenerate", interactive=True)
+ clear_btn = gr.Button(value="🗑️ Clear", interactive=True)
+
+ gr.Markdown(tos_markdown)
+ gr.Markdown(learn_more_markdown)
+ gr.Markdown(ack_markdown)
+
+ regenerate_btn.click(
+ regenerate,
+ [state, image_process_mode],
+ [state, chatbot, textbox, imagebox],
+ queue=False
+ ).then(
+ http_bot,
+ [state, temperature, top_p, max_output_tokens],
+ [state, chatbot]
+ )
+
+ clear_btn.click(
+ clear_history,
+ None,
+ [state, chatbot, textbox, imagebox],
+ queue=False
+ )
+
+ textbox.submit(
+ add_text,
+ [state, textbox, imagebox, image_process_mode],
+ [state, chatbot, textbox, imagebox],
+ queue=False
+ ).then(
+ http_bot,
+ [state, temperature, top_p, max_output_tokens],
+ [state, chatbot]
+ )
+
+ submit_btn.click(
+ add_text,
+ [state, textbox, imagebox, image_process_mode],
+ [state, chatbot, textbox, imagebox],
+ queue=False
+ ).then(
+ http_bot,
+ [state, temperature, top_p, max_output_tokens],
+ [state, chatbot]
+ )
+
+ demo.load(
+ load_demo,
+ None,
+ [state],
+ queue=False
+ )
+ return demo
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--host", type=str, default="0.0.0.0")
+ parser.add_argument("--port", type=int, default=7860)
+ parser.add_argument("--share", default=True)
+ parser.add_argument("--model-path", type=str,
+ default="Lin-Chen/ShareGPT4V-7B")
+ parser.add_argument("--model-name", type=str,
+ default="llava-v1.5-7b")
+ args = parser.parse_args()
+ return args
+
+
+if __name__ == '__main__':
+ args = parse_args()
+ model_name = args.model_name
+ tokenizer, model, image_processor, context_len = load_pretrained_model(
+ args.model_path, None, args.model_name, False, False)
+ demo = build_demo()
+ demo.queue()
+ demo.launch(server_name=args.host,
+ server_port=args.port,
+ share=args.share)