Update Webpage.py

Webpage.py

@@ -6,6 +6,9 @@ import numpy as np
 import cv2 as cv
 import torch
 from PIL import Image
+from gradio_pdf import PDF
+from pdf2image import convert_from_path
+from pathlib import Path
 
 GENERAL_CATEGORY = {'Potatoes / Vegetables / Fruit': 0, 'Chemical products': 1, 'Photo / Film / Optical items': 2, 'Catering industry': 3, 'Industrial products other': 4, 'Media': 5, 'Real estate': 6, 'Government': 7, 'Personnel advertisements': 8, 'Cars / Commercial vehicles': 9, 'Cleaning products': 10, 'Retail': 11, 'Fragrances': 12, 'Footwear / Leather goods': 13, 'Software / Automation': 14, 'Telecommunication equipment': 15, 'Tourism': 16, 'Transport/Communication companies': 17, 'Transport services': 18, 'Insurances': 19, 'Meat / Fish / Poultry': 20, 'Detergents': 21, 'Foods General': 22, 'Other services': 23, 'Banks and Financial Services': 24, 'Office Products': 25, 'Household Items': 26, 'Non-alcoholic beverages': 27, 'Hair, Oral and Personal Care': 28, 'Fashion and Clothing': 29, 'Other products and Services': 30, 'Paper products': 31, 'Alcohol and Other Stimulants': 32, 'Medicines': 33, 'Recreation and Leisure': 34, 'Electronics': 35, 'Home Furnishings': 36, 'Products for Business Use': 37}
 CATEGORIES = list(GENERAL_CATEGORY.keys())
@@ -44,10 +47,13 @@ def calculate_areas(prompts, brand_num, pictorial_num, text_num):
     left_margin = x1; right_margin = w-x2
     if left_margin <=100 and right_margin <= 100:
         upper_margin = y1; lower_margin = h-y2
-        if upper_margin >= lower_margin:
-            context_image = image_entire[:int(y1), :, :]
+        if upper_margin <= 100 and lower_margin <= 100:
+            context_image = None
         else:
-            context_image = image_entire[int(y2):, :, :]
+            if upper_margin >= lower_margin:
+                context_image = image_entire[:int(y1), :, :]
+            else:
+                context_image = image_entire[int(y2):, :, :]
     else:
         if left_margin >= right_margin:
             context_image = image_entire[:, :int(x1), :]
@@ -59,8 +65,14 @@ def calculate_areas(prompts, brand_num, pictorial_num, text_num):
 
     return (brand_surf/whole_size*100, pictorial_surf/whole_size*100, text_surf/whole_size*100, ad_size/whole_size*100, ad_image, context_image)
 
+def convert(note, doc):
+    print(doc)
+    img = convert_from_path(doc)[0]
+    img.save(f'pdf_to_imgs/pdf_img.png', 'PNG')
+    return 'Done!', gr.DownloadButton(label='Download converted image', value='pdf_to_imgs/pdf_img.png')
 
-def attention(notes, download1, download2, whole_display_prompt, 
+def attention(note, button1, button2,
+              whole_display_prompt, 
               brand_num, pictorial_num, text_num,
               category, ad_location, gaze_type):
     text_detection_model_path = 'EAST-Text-Detection/frozen_east_text_detection.pb'
@@ -82,7 +94,10 @@ def attention(notes, download1, download2, whole_display_prompt,
     surfaces = [brand_percent, visual_percent, text_percent, adv_size_percent*10/100]
 
     # caption_ad = XGBoost_utils.Caption_Generation(Image.fromarray(np.uint8(ad_image)))
-    # caption_context = XGBoost_utils.Caption_Generation(Image.fromarray(np.uint8(context_image)))
+    # if context_image is not None:
+    #     caption_context = XGBoost_utils.Caption_Generation(Image.fromarray(np.uint8(context_image)))
+    # else:
+    #     caption_context = ''
     # ad_topic = XGBoost_utils.Topic_emb(caption_ad)
     # ctpg_topic = XGBoost_utils.Topic_emb(caption_context)
     np.random.seed(42)
@@ -91,10 +106,16 @@ def attention(notes, download1, download2, whole_display_prompt,
 
     ad = cv.resize(ad_image, (640, 832))
     print('ad shape: ', ad.shape)
-    context = cv.resize(context_image, (640, 832))
+    if context_image is None:
+        context = None
+    else:
+        context = cv.resize(context_image, (640, 832))
 
     adv_imgs = torch.permute(torch.tensor(ad), (2,0,1)).unsqueeze(0)
-    ctpg_imgs = torch.permute(torch.tensor(context), (2,0,1)).unsqueeze(0)
+    if context is None:
+        ctpg_imgs = torch.zeros_like(adv_imgs)
+    else:
+        ctpg_imgs = torch.permute(torch.tensor(context), (2,0,1)).unsqueeze(0)
     ad_locations = torch.tensor([1,0]).unsqueeze(0)
     heatmap = Predict.HeatMap_CNN(adv_imgs, ctpg_imgs, ad_locations, Gaze_Type='AG')
 
@@ -104,27 +125,50 @@ def attention(notes, download1, download2, whole_display_prompt,
                                     ad_embeddings=ad_topic, ctpg_embeddings=ctpg_topic,
                                     surface_sizes=surfaces, Product_Group=prod_group,
                                     obj_detection_model_pth=None, num_topic=20, Gaze_Time_Type=gaze_type)
-    return np.round(Gaze,2), Image.fromarray(np.flip(heatmap, axis=2)), "Hotter/Redder regions show more pixel contribution."
+    return np.round(Gaze,2), Image.fromarray(np.flip(heatmap, axis=2))
 
 with gr.Blocks() as demo:
+    gr.Markdown("""
+                <div style='text-align: center; padding: 10px; font-size:40px'>
+                    <p> <b>Gazer 1.0: Ad Attention Prediction</b> </p>
+                </div>
+                """)
+    gr.Markdown("""
+                This app accompanies: "Contextual Advertising with Theory-Informed Machine Learning", manuscript submitted to the Journal of Marketing.  
+                App Version: 1.0, Date: 10/24/2024.  
+                Note: Gazer 1.0 does not yet include LLM generated ad topics. Future updates will include this in a GPU environment.
+                """)
+    gr.Interface(
+        fn=convert,
+        inputs=[gr.Markdown("""
+                            <div style='font-size:20px'>
+                                <p> <b>If you only have a pdf image file, first convert it here to png file and download:</b> </p>
+                            </div>
+                           
+                           """),
+                PDF(label="PDF Converter")],
+        outputs=[gr.Text(label='Progress'), gr.DownloadButton(label='Wait to be downloadable', value=None)]
+    )
+    
     gr.Interface(
         fn=attention,
         inputs=[gr.Markdown("""
-                            ### Instruction: 
-                            1. Click to upload or drag the entire image that contains BOTH ad and its context;  
-                            2. Draw bounding boxes in the order of: (each element can have more than 1 boxes; remember the number of boxes for each element you draw)  
-                            &nbsp;&nbsp;&nbsp;(a) Brand element(s) (skip if N.A.)  
-                            &nbsp;&nbsp;&nbsp;(b) Pictorial element(s), e.g. Objects, Person etc (skip if N.A.)  
-                            &nbsp;&nbsp;&nbsp;(c) Text element(s) (skip if N.A.)  
-                            &nbsp;&nbsp;&nbsp;(d) The advertisement.  
-                            3. Put in number of bounding boxes for each element, product category, ad location and attention type.  
-
-                            ***NOTE:*** *ResNet50 Heatmap could take around 20-80 seconds under current CPU environment.*
-                                
-                            Two example ads are avialable for download: """),
+                ## Instructions:  
+                0. The screen size should remain the same during processing.
+                1. Click to upload or drag the entire image (jpg/jpeg/png file) that contains BOTH ad and its context;  
+                2. Draw bounding boxes in the order of: (each element can have more than 1 boxes; remember the number of boxes for each element you draw)  
+                &nbsp;&nbsp;&nbsp;(a) Brand element(s) (skip if N.A.)  
+                &nbsp;&nbsp;&nbsp;(b) Pictorial element(s), e.g. Objects, Person etc (skip if N.A.)  
+                &nbsp;&nbsp;&nbsp;(c) Text element(s) (skip if N.A.)  
+                &nbsp;&nbsp;&nbsp;(d) The advertisement.  
+                3. Put in number of bounding boxes for each element, product category, ad location and attention type.  
+
+                ***NOTE:*** *ResNet50 Heatmap could take around 20-80 seconds under current CPU environment.*
+                    
+                Two example ads are avialable for download: """),
                 gr.DownloadButton(label="Download Example Image 1 of Ad and Context", value='Demo/Ad_Example1.jpg'),
                 gr.DownloadButton(label="Download Example Image 2 of Ad and Context", value='Demo/Ad_Example2.jpg'),
-                ImagePrompter(label="Upload Entire (Ad+Context) Image, and Draw Bounding Boxes", sources=['upload'], type="pil"),
+                ImagePrompter(label="Upload Entire (Ad+Context) Image in jpg/jpeg/png format, and Draw Bounding Boxes", sources=['upload'], type="pil"),
                 gr.Number(label="Number of brand bounding boxes drawn"),
                 gr.Number(label="Number of pictorial bounding boxes drawn"),
                 gr.Number(label="Number of text bounding boxes drawn"),
@@ -133,12 +177,9 @@ with gr.Blocks() as demo:
                 gr.Dropdown(GAZE_TYPE, label='Gaze Type')
                 ],
         outputs=[gr.Number(label="Predicted Gaze (sec)"),
-                 gr.Image(label="ResNet50 Heatmap"),
-                 gr.Textbox(label="Heatmap Info")],
-        title="""Gazer 1.0: Ad Attention Prediction""",
-        description="""This app accompanies: "Contextual Advertising with Theory-Informed Machine Learning", manuscript submitted to the Journal of Marketing.  
-                       App Version: 1.0, Date: 10/24/2024.  
-                       Note: Gazer 1.0 does not yet include LLM generated ad topics. Future updates will include this in a GPU environment.""",
+                 gr.Image(label="ResNet50 Heatmap (Hotter/Redder regions show more pixel contribution.)")],
+        title=None,
+        description=None,
         theme=gr.themes.Soft()
     )
     gr.Markdown(
@@ -149,4 +190,4 @@ with gr.Blocks() as demo:
             """
         )
 
-demo.launch(share=True)
+demo.launch(share=False)