Update README.md

README.md

@@ -51,11 +51,11 @@ from transformers.generation import LogitsProcessorList, PrefixConstrainedLogits
 import torch
 
 import sys
-sys.path.append(PATH_TO_BAAI_Emu3-Gen_MODEL)
+sys.path.append(PATH_TO_BAAI_Emu3-Stage1_MODEL)
 from processing_emu3 import Emu3Processor
 
 # model path
-EMU_HUB = "BAAI/Emu3-Gen"
+EMU_HUB = "BAAI/Emu3-Stage1"
 VQ_HUB = "BAAI/Emu3-VisionTokenizer"
 
 # prepare model and processor
@@ -67,11 +67,12 @@ model = AutoModelForCausalLM.from_pretrained(
     trust_remote_code=True,
 )
 
-tokenizer = AutoTokenizer.from_pretrained(EMU_HUB, trust_remote_code=True)
+tokenizer = AutoTokenizer.from_pretrained(EMU_HUB, trust_remote_code=True, padding_side="left")
 image_processor = AutoImageProcessor.from_pretrained(VQ_HUB, trust_remote_code=True)
 image_tokenizer = AutoModel.from_pretrained(VQ_HUB, device_map="cuda:0", trust_remote_code=True).eval()
-processor = Emu3Processor(image_processor, image_tokenizer, tokenizer)
+processor = Emu3Processor(image_processor, image_tokenizer, tokenizer, chat_template="{image_prompt}{text_prompt}")
 
+# Image Generation
 # prepare input
 POSITIVE_PROMPT = " masterpiece, film grained, best quality."
 NEGATIVE_PROMPT = "lowres, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry."
@@ -126,4 +127,29 @@ for idx, im in enumerate(mm_list):
         continue
     im.save(f"result_{idx}.png")
 
+
+# Multimodal Understanding
+text = "The image depicts "
+image = Image.open("assets/demo.png")
+
+inputs = processor(
+    text=text,
+    image=image,
+    mode='U',
+    padding="longest",
+    return_tensors="pt",
+)
+GENERATION_CONFIG = GenerationConfig(pad_token_id=tokenizer.pad_token_id, bos_token_id=tokenizer.bos_token_id, eos_token_id=tokenizer.eos_token_id)
+
+ outputs = model.generate(
+     inputs.input_ids.to("cuda:0"),
+     GENERATION_CONFIG,
+     max_new_tokens=1024,
+     attention_mask=inputs.attention_mask.to("cuda:0"),
+ )
+ outputs = outputs[:, inputs.input_ids.shape[-1]:]
+ answers = processor.batch_decode(outputs, skip_special_tokens=True)
+ for ans in answers:
+     print(ans)
+
 ```