environment setup version 1

app.py

@@ -0,0 +1,119 @@
+
+import gradio as gr
+import torch
+import clip
+from PIL import Image
+import os
+import numpy as np
+from matplotlib import pyplot as plt
+from io import BytesIO
+
+# Load CLIP model
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model, preprocess = clip.load("ViT-B/32", device=device)
+
+
+f = open("debug.txt", "w+")
+
+# Examples for Zero-Shot Classification
+classification_examples = [
+    ["example_images/turkish_tea.jpeg", "turkish tea, coffee, water"]
+]
+
+# Examples for Image Retrieval
+gallery_image_paths_list = os.listdir("gallery")
+gallery_image_paths_list = ["gallery/"+path for path in gallery_image_paths_list]
+f.write(str(gallery_image_paths_list))
+retrieval_examples = [
+    [gallery_image_paths_list, "example_images/banana_73.jpg"]
+]
+
+# Zero-shot classification function
+def zero_shot_classification(image, classnames):
+    classnames = [cls.strip() for cls in classnames.split(",")]
+    text_inputs = clip.tokenize(classnames).to(device)
+   
+    img_processed = preprocess(image).unsqueeze(0).to(device)
+    with torch.no_grad():
+        logits_per_image, _ = model(img_processed, text_inputs)
+        probs = logits_per_image.softmax(dim=-1).cpu().numpy()[0]
+
+    results = {classnames[i]: probs[i] for i in range(len(classnames))}
+
+    return results
+
+# Image retrieval function
+def image_retrieval(gallery, query):
+    query_processed = preprocess(query).unsqueeze(0).to(device)
+    query_embedding = model.encode_image(query_processed)
+    query_embedding /= query_embedding.norm(dim=-1, keepdim=True)
+
+    rank_list = []
+
+    for img in gallery:
+       # img = Image.open(img[0])
+        img_processed = preprocess(img[0]).unsqueeze(0).to(device)
+        embedding = model.encode_image(img_processed)
+        embedding /= embedding.norm(dim=-1, keepdim=True)
+
+        similarity_score = (100.0 * query_embedding @ embedding.t()).item()
+        similarity_score = round(similarity_score,3)
+        rank_list.append([similarity_score, img[0]])
+
+
+    rank_list = sorted(rank_list, key=lambda x: x[0], reverse = True)
+
+
+    fig = plt.figure(figsize=(5,20))
+    plot_length = 11
+
+    for i in range(1,plot_length): 
+        gallery_ax = fig.add_subplot(plot_length,1,i)
+        img = rank_list[i][1]
+        gallery_ax.imshow(img)
+        gallery_ax.set_title('%.3f'% rank_list[i][0], fontsize=10) #add similarity score as title
+        gallery_ax.axis('off')
+
+    # Return the top-10 images with their similarity scores
+    buf = BytesIO()
+    fig.savefig(buf, format="png", bbox_inches="tight")  # Save figure to buffer
+    buf.seek(0)
+    img = Image.open(buf)  # Open buffer as PIL Image
+    return img
+
+# Define Gradio interface for zero-shot classification
+classification_interface = gr.Interface(
+    fn=zero_shot_classification,
+    inputs=[
+        gr.Image(type="pil", label="Input Image", sources=["upload"]),
+        gr.Textbox(lines=3, placeholder="Enter labels separated by commas, e.g., dog, cat, car", label="Class Labels"),
+    ],
+    examples=classification_examples,
+    outputs=gr.Label(label="Classification Probabilities"),
+)
+
+# Define Gradio interface for image retrieval
+retrieval_interface = gr.Interface(
+    fn=image_retrieval,
+    inputs=[
+        gr.Gallery(label="Gallery Folder", type="pil", columns=[3], rows=[10], object_fit="contain", height="auto"),
+        gr.Image(type="pil", label="Query Image"),
+    ],
+    outputs= gr.Image(type="pil", label="Top-10 Retrieved Images"),
+    examples=retrieval_examples,
+)
+
+# Combine the interfaces into a single Gradio app
+app = gr.Blocks()
+
+with app:
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown("## Zero-shot Classification")
+            classification_interface.render()
+        with gr.Column():
+            gr.Markdown("## Image Retrieval")
+            retrieval_interface.render()
+
+# Launch the app
+app.launch()

requirements.txt

@@ -0,0 +1,8 @@
+torch==1.13.1
+torchvision==0.14.1
+torchaudio==0.13.1
+gradio
+clip-by-openai
+pillow
+matplotlib
+ftfy