Spaces:
Runtime error
Runtime error
File size: 5,455 Bytes
fb740cb f3eb930 fb740cb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 |
import gradio as gr
import torch.nn.functional as F
# from PIL import Image
import uform
model = uform.get_model('unum-cloud/uform-vl-english')
def find_score(img, txt, if_fine_grained):
txt = model.preprocess_text(txt)
img = model.preprocess_image(img)
txt_features, txt_emb = model.encode_text(txt, return_features = True)
img_features, img_emb = model.encode_image(img, return_features = True)
if if_fine_grained:
joint_embedding = model.encode_multimodal(
image_features=img_features,
text_features=txt_features,
attention_mask=txt['attention_mask'])
score = model.get_matching_scores(joint_embedding)
else:
score = F.cosine_similarity(txt_emb, img_emb)
return score
def find_score_img(img1, img2, if_fine_grained):
img1 = model.preprocess_image(img1)
img2 = model.preprocess_image(img2)
img_features1, img_emb1 = model.encode_image(img1, return_features = True)
img_features2, img_emb2 = model.encode_image(img2, return_features = True)
if if_fine_grained:
joint_embedding = model.encode_multimodal(
image_features=img_features1,
text_features=img_features2,
attention_mask=img1['attention_mask'])
score = model.get_matching_scores(joint_embedding)
else:
score = F.cosine_similarity(img_emb1, img_emb2)
return score
def find_score_txt(txt1, txt2, if_fine_grained):
txt1 = model.preprocess_text(txt1)
txt2 = model.preprocess_text(txt2)
txt_features1, txt_emb1 = model.encode_text(txt1, return_features = True)
txt_features2, txt_emb2 = model.encode_text(txt2, return_features = True)
if if_fine_grained:
joint_embedding = model.encode_multimodal(
image_features=txt_features1,
text_features=txt_features2,
attention_mask=txt1['attention_mask'])
score = model.get_matching_scores(joint_embedding)
else:
score = F.cosine_similarity(txt_emb1, txt_emb2)
return score
with gr.Blocks(theme = gr.themes.Glass()) as demo_mix:
gr.Markdown('# Find similarity between images and text.')
with gr.Row():
with gr.Column():
img_input = gr.Image(source = 'upload', type = 'pil', label = "Drop your image here", shape = [256, 256])
with gr.Column():
txt_input = gr.Textbox(label = 'Enter your text here:' , lines = 1,)
if_fine_grained = gr.Checkbox(label = "Check for a more fine-grained comparison")
btn = gr.Button("Find similarity")
score = gr.Number(label='Similarity score')
btn.click(find_score, inputs=[img_input, txt_input, if_fine_grained], outputs=[score])
gr.Markdown('If the box for a more fine-grained comparison is checked, the similarity score will be in (0,1), otherwise, it will be in (-1,1)')
gr.Markdown('### Image examples')
gr.Examples(['imgs/red_panda.jpg', 'imgs/trash_raccoon.jpg', 'imgs/baby_panda.jpg', 'imgs/rocket.jpg'], inputs=[img_input])
gr.Markdown('### Text examples')
gr.Examples(['baby red panda staring into the camera', \
'trash raccoon peeking from a trash bin',
'a cartoonish raccoon wearing a blue and red suit',
"a person holding a baby panda"], inputs=[txt_input])
with gr.Blocks() as demo_img:
gr.Markdown('# Find similarity between images.')
with gr.Row():
with gr.Column():
img_input1 = gr.Image(source = 'upload', type = 'pil', label = "Drop your image here", shape = [256, 256])
if_fine_grained = gr.Checkbox(label = "Check for a more fine-grained comparison")
with gr.Column():
img_input2 = gr.Image(source = 'upload', type = 'pil', label = "Drop your image here", shape = [256, 256])
btn = gr.Button("Find similarity")
score = gr.Number(label='Similarity score')
btn.click(find_score_img, inputs=[img_input1, img_input2, if_fine_grained], outputs=[score])
gr.Markdown('If the box for a more fine-grained comparison is checked, the similarity score will be in (0,1), otherwise, it will be in (-1,1)')
gr.Markdown('### Image examples')
gr.Examples(['imgs/red_panda.jpg', 'imgs/trash_raccoon.jpg', 'imgs/baby_panda.jpg', 'imgs/rocket.jpg'], inputs=[img_input1])
with gr.Blocks() as demo_txt:
gr.Markdown('# Find similarity between short descriptions.')
with gr.Row():
with gr.Column():
txt_input1 = gr.Textbox(label = 'Enter your text here:' , lines = 1)
txt_input2 = gr.Textbox(label = 'Enter your text here:' , lines = 1)
with gr.Column():
if_fine_grained = gr.Checkbox(label = "Check for a more fine-grained comparison")
btn = gr.Button("Find similarity")
score = gr.Number(label='Similarity score')
btn.click(find_score_txt, inputs=[img_input, txt_input, if_fine_grained], outputs=[score])
gr.Markdown('If the box for a more fine-grained comparison is checked, the similarity score will be in (0,1), otherwise, it will be in (-1,1)')
gr.Markdown('### Text examples')
gr.Examples(['baby red panda staring into the camera', \
'trash raccoon peeking from a trash bin',
'a cartoonish raccoon wearing a blue and red suit',
"a person holding a baby panda"], inputs=[txt_input1])
demo = gr.TabbedInterface([demo_mix, demo_img, demo_txt], ["img2txt", "img2img", "txt2txt"])
if __name__ == "__main__":
demo.launch(share=True) |