ov_mllama_generator_script.py

""" Main inference generation for mLlama-3.2-11B compressed and packaged as OV model

    -- accompanying generator_class file - ov_mllama_generator_class.py

    -- dependencies:  transformers and torch

"""

import requests
import openvino as ov

from PIL import Image
from transformers import TextStreamer, AutoProcessor
import numpy as np

from ov_mllama_generator_class import OVMLlamaForConditionalGeneration

model_id = "meta-llama/Llama-3.2-11B-Vision-Instruct"
model_dir = "C:\\Users\\darre\\llmware_data\\model_repo\\llama-11b-vision-instruct-ov"

core = ov.Core()

language_model_name = "llm_int4_asym_r10_gs64_max_activation_variance_scale_all_layers.xml"
vision_encoder_name = "openvino_vision_encoder_int8.xml"
device="CPU"

ov_model = OVMLlamaForConditionalGeneration(model_dir, device=device,
                                            language_model_name=language_model_name,
                                            image_encoder_name=vision_encoder_name)

processor = AutoProcessor.from_pretrained(model_dir)

question = "What is unusual on this image?"

messages = [
    {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": question}]},
]
text = processor.tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
url = "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11"
raw_image = Image.open(requests.get(url, stream=True).raw)

inputs = processor(text=text, images=[raw_image], return_tensors="pt")
streamer = TextStreamer(processor.tokenizer, skip_prompt=True, skip_special_tokens=True)
print(f"Question: {question}")

output = ov_model.generate(**inputs, do_sample=False, max_new_tokens=100, temperature=None, top_p=None, streamer=streamer)
print(f"Visual encoder time {ov_model.vision_encoder_infer_time[0] * 1000 :.2f} ms")
print(f"First token latency {ov_model.llm_infer_time[0] * 1000 :.2f}ms, Second token latency {np.mean(np.array(ov_model.llm_infer_time[1:])) * 1000:.2f}ms")