Context-Informed-Descriptions

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.9
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY . .
+
+CMD ["gunicorn", "-b", "0.0.0.0:7860", "main:app", ]

main.py

@@ -0,0 +1,51 @@
+import json
+import tiktoken
+import requests
+from flask_cors import CORS, cross_origin
+from PIL import Image
+from transformers import CLIPProcessor, CLIPModel, pipeline, AutoTokenizer
+from flask import Flask, request, jsonify
+
+app = Flask(__name__)
+CORS(app)
+
+encoding = tiktoken.get_encoding("cl100k_base")
+model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
+tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large-cnn")
+processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
+summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
+
+@app.route("/")
+
+@app.route('/clip-scores', methods=['POST'])
+@cross_origin(origin='chrome-extension://caglaokjfpffmkcbjknpolcjgjomamea') 
+def getClipScores():
+   if request.method == 'POST':
+        data = request.get_json()
+        scores = data.get('scores')
+        imageUrl = data.get('imageUrl')
+        image = Image.open(requests.get(imageUrl, stream=True).raw)
+        for entry in scores:
+            entry["summary"] = summarize_text(entry["text"])
+        texts = [entry['summary'] for entry in scores]
+        inputs = processor(text=texts, images=image, return_tensors="pt", truncation=True, padding=True)
+        outputs = model(**inputs)
+        logits_per_image = outputs.logits_per_image
+        probs = logits_per_image.softmax(dim=1).detach().cpu().numpy()
+        for i, entry in enumerate(scores):
+            entry['clip_score'] = float(probs[0, i])
+        sorted_scores = sorted(scores, key=lambda x: x['clip_score'], reverse= True)
+        print(jsonify("Sorted scores json", sorted_scores))
+        return jsonify(sorted_scores) 
+
+def summarize_text (text):
+    tokens = tokenizer(text, return_tensors='pt', truncation=True, max_length=tokenizer.model_max_length, padding="max_length")
+    input_ids = tokens.input_ids
+    if input_ids.size(1) > tokenizer.model_max_length:
+        summary = summarizer(text, max_length=77, min_length=20, do_sample=False)
+        return summary[0]['summary_text']
+    else:
+        return text
+    
+if __name__ == "__main__":
+    app.run(debug = True)

requirements.txt

@@ -0,0 +1,2 @@
+flask
+gunicorn