added 2 passes

app.py

@@ -1,36 +1,63 @@
 import streamlit as st
-from PIL import Image
-from utils import get_gpt4V_response, get_str_to_json
-
-
 st.set_page_config(page_title="GPT-4V Demo", page_icon="🧠", layout="wide")
 
+from PIL import Image
+import base64
+from io import BytesIO
+from utils import get_str_to_json
+from pass1 import get_gpt4V_response_1
+from pass2 import get_gpt4V_response_2
+from examples import example_1, example_2
+
+def clear_data():
+    st.session_state["story"] = ""
+    st.session_state["goal"] = ""
+    st.session_state["entity"] = ""
+    st.session_state["images"] = []
+    for key in st.session_state.keys():
+        st.session_state.pop(key)
+    # st.rerun()
+
+print(st.session_state)
+
 with st.sidebar:
+    if st.button("Clear Inputs"):
+        clear_data()
+
     st.title("Parameters")
     st.write("This is a demo of GPT-4V model. It takes a story, goal, entity and an image as input and generates a response.")
 
     st.subheader("Sampling Temperature")
-    temperature = st.slider(label="", min_value=0.1, max_value=1.0, value=0.5, step=0.1)
+    temperature = st.slider(label="x", min_value=0.1, max_value=1.0, value=0.5, step=0.1, label_visibility='hidden')
     st.write("The sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.")
 
     st.subheader("Entity?")
-    entity_opt = st.radio(label="With or Without", options=[1, 0], format_func=lambda x: ["Without", "With"][x])
+    entity_opt = st.radio(label="With or Without", options=[1, 0], format_func=lambda x: ["Without", "With"][x], on_change=clear_data)
 
+    st.subheader("Examples")
+    cols = st.columns(2)
+    for i, example in enumerate([example_1, example_2]):
+        with cols[i % len(cols)]:
+            if st.button(f"Example {i+1}", key=f"example{i+1}"):
+                clear_data()
+                st.session_state["data"] = example
 
 def main():
     global temperature, entity
     st.title('What can go wrong?')
 
+    data = st.session_state.get("data", None)
+
     col1, col2 = st.columns(2)
 
     with col1:
-        story = st.text_area("Story", placeholder="Enter the story here")
+        story = st.text_area("Story", placeholder="Enter the story here", value=(data.story if data else ""), key="story")
 
         entity = None
         if entity_opt:
-            entity = st.text_input("Entity", placeholder="Enter the entity here")
+            entity = st.text_input("Entity", placeholder="Enter the entity here", value=(data.entity if data else ""), key="entity")
 
-        goal = st.text_area("Goal", placeholder="Enter the goal here")
+        goal = st.text_area("Goal", placeholder="Enter the goal here", value=(data.goal if data else ""), key="goal")
 
         images = st.file_uploader("Upload Image", type=['jpg', 'png'], accept_multiple_files=True)
 
@@ -41,35 +68,100 @@ def main():
                 with cols[i]:
                     image = Image.open(image)
                     st.image(image, caption="Uploaded Image", use_column_width=True)
-        
-
-        if st.button("Submit"):
-            if not story or not goal or (entity_opt and not entity) or not images:
+        elif not images and data:
+            cols = st.columns(len(data.images))
+            for i, imb64 in enumerate(data.images_base64):
+                with cols[i]:
+                    image = Image.open(BytesIO(base64.b64decode(imb64)))
+                    st.image(image, caption="Example Image", use_column_width=True)
+
+        if st.button("Pass 1"):
+            st.session_state["button_2"] = False
+            image_to_send = None
+            if images:
+                image_to_send = images
+            elif data:
+                image_to_send = data.images_base64
+
+            if not story or not goal or (entity_opt and not entity) or not image_to_send:
                 st.error("Please fill all the fields")
                 return
             with col2:
                 with st.status("Generating response...", expanded=True):
-                    response = get_gpt4V_response(story, goal, entity, images, temperature=temperature)
+                    response = get_gpt4V_response_1(story, goal, entity, image_to_send, temperature=temperature)
                 
+                response_json = {}
                 try:
                     response_json = get_str_to_json(response)
                     if "condition" not in response_json or "alternate_condition" not in response_json:
                         raise ValueError("Invalid JSON - 1")
                     if not entity_opt and "entity" not in response_json:
                         raise ValueError("Invalid JSON - 2")
+
+                except Exception as e:
+                    print("Exception 1", e)
+                    response_json = {
+                        "entity": "",
+                        "condition": "",
+                        "alternate_condition": "",
+                        "response": response
+                    }
+
+                finally:
+                    out1 = {
+                        "entity": response_json.get("entity", None),
+                        "condition": response_json.get("condition", None),
+                        "alternate_condition": response_json.get("alternate_condition", None),
+                        "response": response_json.get("response", "")
+                    }
+                    st.session_state["output_1"] = out1
+                    st.session_state["button_1"] = True
+    
+    with col2:
+        if st.session_state.get("button_1", False): # If pass 1 is done
+            output_1 = st.session_state.get("output_1", {})
+
+            if "response" in output_1 and output_1["response"]:
+                st.warning(f"Failed to parse JSON. Going for full output")
+                st.write(output_1["response"])
+
+            entity = output_1.get("entity", "")
+            condition = output_1.get("condition", "")
+            alternate_condition = output_1.get("alternate_condition", "")
+            if not entity_opt:
+                st.text_input("Entity", value=entity)
+            st.text_area("Condition", value=condition)
+            st.text_area("Alternate Condition", value=alternate_condition)
+
+            if st.button("Pass 2"):
+                st.session_state["button_2"] = True
+
+                with st.status("Generating response...", expanded=True):
+                    response = get_gpt4V_response_2(story, goal, alternate_condition, images, temperature=temperature)
+
+                try:
                     
-                    if not entity_opt:
-                        with st.expander("Entity", expanded=True):
-                            st.write(response_json["entity"])
-                    
-                    with st.expander("Condition", expanded=True):
-                        st.write(response_json["condition"])
-                    
-                    with st.expander("Alternate Condition", expanded=True):
-                        st.write(response_json["alternate_condition"])
+                    response_json = get_str_to_json(response)
+
+                    if "event" not in response_json:
+                        raise ValueError("Invalid JSON - 3")
 
                 except Exception as e:
-                    print(e)
+                    print("Exception 2", e)
                     st.warning(f"Failed to parse JSON. Going for full output")
-                    st.write(response)
+                    response_json = {
+                        "event": response
+                    }
+
+                finally:
+                    out2 = {
+                        "event": response_json.get("event", response)
+                    }
+                    st.session_state["output_2"] = out2
+
+        if st.session_state.get("button_2", False): # If pass 2 is done
+            output_2 = st.session_state.get("output_2", {})
+            st.subheader("Event Leads to Alternate Condition")
+            st.write(output_2.get("event", ""))
+
 main()

examples.py

@@ -0,0 +1,52 @@
+import streamlit as st
+import requests
+import base64
+
+
+example_1 = {
+    "story": "The parade is starting and people line the streets. The folks who live upstairs in the buildings get a good view. The really cool floats are coming next. It's raining harder but no one seems to mind. The people came prepared with umbrellas for this special occasion.The more the parade goes on, the better the displays seem to be. The zoo animals are very well done and everybody is standing in the rain to see them. The man's dog is a bit confused by these animals. Everyone is having a good time watching the parade and chatting with their neighbors.",
+    "entity": "Rhinoceros - on a float in the street",
+    "goal": "Watch a parade from the high vantage point.",
+    "images": [
+        "https://drive.google.com/uc?export=view&id=1LhLpWAkuIIVDf5jl1E7_DppoHQwl-7P9",
+        "https://drive.google.com/uc?export=view&id=11r_tEEa7QcEkalT_wcZOtOxnaztqtETh",
+        "https://drive.google.com/uc?export=view&id=1NsZpx_otGbh9Jz00A9eyF3xoUfTnvNxH"
+    ]
+}
+
+example_2 = {
+    "story": "There is a place in the world where the most people go to in their holidays they visit the biggest desert where people gets into helicopter and sees the enormous land from above and the warm air they you feel and there are other people who just like to travel by travel by walking with your family and friends.There is a ancient history with that log old cowboys of the western side used to camp there and have fum stories with their family and friends but one day a disaster happened which caused the log get red and rotten and from all over there world tourist goes to that wonderful place.",
+    "entity": "Sand",
+    "goal": "See a beautiful desert landscape.",
+    "images": [
+        "https://drive.google.com/uc?export=view&id=1DyIOAuMsSy1flFmvB255cT5F3ScI2qfd",
+        "https://drive.google.com/uc?export=view&id=1rn7_WOGfwMoUurHog8DQV5GEIA0kugGB",
+        "https://drive.google.com/uc?export=view&id=112TxpwMDA34EO0Rrleh7XnBG56YN0vHt"
+    ]
+}
+
+class Example:
+    def __init__(self, story, entity, goal, images):
+        self.story = story
+        self.entity = entity
+        self.goal = goal
+        self.images = images
+        self.images_base64 = self.get_images()
+
+    @st.cache_data
+    def download_image(_self, image_url):
+        return requests.get(image_url).content
+
+    @st.cache_data
+    def get_base64(_self, img_data):
+        return base64.b64encode(img_data)
+    
+    @st.cache_data
+    def get_images(self):
+        return [self.get_base64(self.download_image(image_url)) for image_url in self.images]
+
+
+
+example_1 = Example(example_1["story"], example_1["entity"], example_1["goal"], example_1["images"])
+
+example_2 = Example(example_2["story"], example_2["entity"], example_2["goal"], example_2["images"])

pass1.py

@@ -0,0 +1,99 @@
+import streamlit as st
+from utils import im_2_b64, RANDOM_SEED, client
+
+
+with_prompt = """
+In the following task, you will be presented with some images and a story that is, in some manner, related to that goal. You will be given a specific  goal and entity (generally, person or object), and be asked to identify condition necessary for that goal and the alternate condition that could prevent that goal.
+
+Conditions for the output:
+1. Condition: The condition is the necessary condition for the goal to be achieved. If the condition is not met, the goal cannot be achieved. The condition must be related to the entity.
+
+2. Alternate Condition: The alternate condition is a different version of the condition that would prevent the goal from being achieved. It is likely that this alternate condition will contradict information provided in the images and/or story.
+
+Output in a python dictionary where it should have the following keys: 'condition', 'alternate_condition'.
+
+Story: {story}
+
+Entity: {entity}
+
+Goal: {goal}
+"""
+
+wo_prompt = """
+In the following task, you will be presented with a image and a story that is, in some manner, related to that goal. You will be given a specific  goal, and be asked to identify an entity (person or object), condition necessary for that goal and the alternate condition that could prevent that goal.
+
+Conditions for the output:
+1. Entity: The entity is the person or object that the goal is related to. The entity should be a crucial part for achieving the goal.
+
+2. Condition: The condition is the necessary condition for the goal to be achieved. If the condition is not met, the goal cannot be achieved. The condition must be related to the entity.
+
+3. Alternate Condition: The alternate condition is a different version of the condition that would prevent the goal from being achieved. It is likely that this alternate condition will contradict information provided in the images and/or story.
+
+Output in a python dictionary where it should have the following keys: 'entity', 'condition', 'alternate_condition'.
+
+Story: {story}
+
+Goal: {goal}
+"""
+
+def get_gpt4V_response_1(story, goal, entity, images, temperature=0.5):
+    # Convert image to base64
+    image_urls = []
+    for image in images:
+        if type(images[0]) == bytes:
+            image_b64 = image
+            image_url = f"data:image/jpeg;base64,{image_b64.decode('utf-8')}"
+        else:
+            image_b64 = im_2_b64(image)
+            image_url = f"data:image/jpeg;base64,{image_b64.decode('utf-8')}"
+        image_urls.append(image_url)
+    st.write("✅ Image converted")
+
+    if entity:
+        prompt = with_prompt
+        now_prompt = prompt.format(story=story, goal=goal, entity=entity)
+    else:
+        prompt = wo_prompt
+        now_prompt = prompt.format(story=story, goal=goal)
+
+    content = [
+        {"type": "text", "text": now_prompt},
+    ]
+
+    st.write("✅ Prompt created")
+
+    for image_url in image_urls:
+        content.append({
+            "type": "image_url",
+            "image_url": {
+                "url": image_url,
+            },
+        })
+    
+    st.write("🚀 Getting Response from GPT4V")
+    response = client.chat.completions.create(
+        model="gpt-4-vision-preview",
+        seed=RANDOM_SEED,
+        messages=[
+            {
+                "role": "user",
+                "content": content
+            }
+        ],
+        temperature=temperature,
+        max_tokens=1024,
+        # top_p=1,
+        # frequency_penalty=0,
+        # presence_penalty=0,
+    )
+    # print(response)
+    # print("Prompt:")
+    # print(now_prompt)
+    out = response.choices[0].message.content
+    # print("OUTPUT:", out)
+    # print("====================================")
+    # print()
+
+    st.write("✅ Response generated")
+
+    return out

pass2.py

@@ -0,0 +1,71 @@
+from utils import im_2_b64, RANDOM_SEED, client
+import streamlit as st
+
+
+prompt = """
+In the following task, you will be presented with some images and a story that is, in some manner, related to that goal. You will be given a specific  goal, and a condition that could prevent that goal from being achieved. Using both story and images, you will be asked to identify an event that could result in that condition being true thus preventing goal success. If it is not plausible to identify such an event, you can state that as well.
+
+Output in a python dictionary where it should have the following keys: 'event'.
+
+Story: {story}
+
+Goal: {goal}
+
+Condition: {condition}
+"""
+
+def get_gpt4V_response_2(story, goal, condition, images, temperature=0.5):
+    # Convert image to base64
+    image_urls = []
+    for image in images:
+        if type(images[0]) == bytes:
+            image_b64 = image
+            image_url = f"data:image/jpeg;base64,{image_b64.decode('utf-8')}"
+        else:
+            image_b64 = im_2_b64(image)
+            image_url = f"data:image/jpeg;base64,{image_b64.decode('utf-8')}"
+        image_urls.append(image_url)
+    st.write("✅ Image converted")
+
+    now_prompt = prompt.format(story=story, goal=goal, condition=condition)
+
+    content = [
+        {"type": "text", "text": now_prompt},
+    ]
+
+    st.write("✅ Prompt created")
+
+    for image_url in image_urls:
+        content.append({
+            "type": "image_url",
+            "image_url": {
+                "url": image_url,
+            },
+        })
+    
+    st.write("🚀 Getting Response from GPT4V")
+    response = client.chat.completions.create(
+        model="gpt-4-vision-preview",
+        seed=RANDOM_SEED,
+        messages=[
+            {
+                "role": "user",
+                "content": content
+            }
+        ],
+        temperature=temperature,
+        max_tokens=1024,
+        # top_p=1,
+        # frequency_penalty=0,
+        # presence_penalty=0,
+    )
+    # print(response)
+    # print("Prompt:")
+    # print(now_prompt)
+    out = response.choices[0].message.content
+    # print("OUTPUT:", out)
+    # print("====================================")
+    # print()
+
+    st.write("✅ Response generated")
+    return out

utils.py

@@ -20,112 +20,6 @@ def im_2_b64(image):
 RANDOM_SEED = 42
 client = OpenAI(api_key=st.secrets["OPENAI_KEY"])
 
-
-with_prompt = """
-In the following task, you will be presented with a image and a story that is, in some manner, related to that goal. You will be given a specific  goal and entity (generally, person or object), and be asked to identify condition necessary for that goal and the alternate condition that could prevent that goal.
-
-Conditions for the output:
-1. Condition: The condition is the necessary condition for the goal to be achieved. If the condition is not met, the goal cannot be achieved. The condition must be related to the entity.
-
-2. Alternate Condition: The alternate condition is a different version of the condition that would prevent the goal from being achieved. It is likely that this alternate condition will contradict information provided in the images and/or story.
-
-Output in a python dictionary where it should have the following keys: 'condition', 'alternate_condition'.
-
-Story: {story}
-
-Entity: {entity}
-
-Goal: {goal}
-"""
-
-wo_prompt = """
-In the following task, you will be presented with a image and a story that is, in some manner, related to that goal. You will be given a specific  goal, and be asked to identify an entity (person or object), condition necessary for that goal and the alternate condition that could prevent that goal.
-
-Conditions for the output:
-1. Entity: The entity is the person or object that the goal is related to. The entity should be a crucial part for achieving the goal.
-
-2. Condition: The condition is the necessary condition for the goal to be achieved. If the condition is not met, the goal cannot be achieved. The condition must be related to the entity.
-
-3. Alternate Condition: The alternate condition is a different version of the condition that would prevent the goal from being achieved. It is likely that this alternate condition will contradict information provided in the images and/or story.
-
-Output in a python dictionary where it should have the following keys: 'entity', 'condition', 'alternate_condition'.
-
-Story: {story}
-
-Goal: {goal}
-"""
-
-data = {
-    "Story id": [],
-    "Prompt": [],
-    "entity": [],
-    "agent": [],
-    "story": [],
-    "Image1": [],
-    "Image2": [],
-    "Image3": [],
-    "GPT-4 Output": [],
-}
-
-
-def get_gpt4V_response(story, goal, entity, images, temperature=0.5):
-    # Convert image to base64
-    image_urls = []
-    for i, image in enumerate(images):
-        image_b64 = im_2_b64(image)
-        image_url = f"data:image/jpeg;base64,{image_b64.decode('utf-8')}"
-        image_urls.append(image_url)
-    st.write("✅ Image converted")
-
-    if entity:
-        prompt = with_prompt
-        now_prompt = prompt.format(story=story, goal=goal, entity=entity)
-    else:
-        prompt = wo_prompt
-        now_prompt = prompt.format(story=story, goal=goal)
-
-    content = [
-        {"type": "text", "text": now_prompt},
-    ]
-
-    st.write("✅ Prompt created")
-
-    for image_url in image_urls:
-        content.append({
-            "type": "image_url",
-            "image_url": {
-                "url": image_url,
-            },
-        })
-    
-    st.write("🚀 Getting Response from GPT4V")
-    response = client.chat.completions.create(
-        model="gpt-4-vision-preview",
-        seed=RANDOM_SEED,
-        messages=[
-            {
-                "role": "user",
-                "content": content
-            }
-        ],
-        temperature=temperature,
-        max_tokens=1024,
-        # top_p=1,
-        # frequency_penalty=0,
-        # presence_penalty=0,
-    )
-    print(response)
-    print("Prompt:")
-    print(now_prompt)
-    out = response.choices[0].message.content
-    print("OUTPUT:", out)
-    print("====================================")
-    print()
-
-    st.write("✅ Response generated")
-
-    return out
-
 def get_str_to_json(st):
     st = re.sub(r"```python", "", st)
     st = re.sub(r"```", "", st)