intital changes to website

index.html

@@ -33,34 +33,26 @@
     <div class="container is-max-desktop">
       <div class="columns is-centered">
         <div class="column has-text-centered">
-          <h1 class="title is-1 publication-title">Nerfies: Deformable Neural Radiance Fields</h1>
+          <h1 class="title is-1 publication-title">MLLMs Know Where to Look:<br>Training-Free Perception of Small Visual Details with Multimodal LLMs</h1>
           <div class="is-size-5 publication-authors">
             <span class="author-block">
-              <a href="https://keunhong.com" target="_blank">Keunhong Park</a><sup>1</sup>,</span>
+              <a href="https://saccharomycetes.github.io/" target="_blank">Jiarui Zhang </a><img src="./static/images/usc_logo.png" style="height: 1em; vertical-align: middle;">,</span>
             <span class="author-block">
-              <a href="https://utkarshsinha.com" target="_blank">Utkarsh Sinha</a><sup>2</sup>,</span>
+              <a href="https://mahyarkoy.github.io/" target="_blank">Mahyar Khayatkhoei </a><img src="./static/images/usc_logo.png" style="height: 1em; vertical-align: middle;">,</span>
             <span class="author-block">
-              <a href="https://jonbarron.info" target="_blank">Jonathan T. Barron</a><sup>2</sup>,
+              <a href="https://www.prateekchhikara.com/" target="_blank">Prateek Chhikara </a><img src="./static/images/usc_logo.png" style="height: 1em; vertical-align: middle;">,
             </span>
+            and
             <span class="author-block">
-              <a href="http://sofienbouaziz.com" target="_blank">Sofien Bouaziz</a><sup>2</sup>,
-            </span>
-            <span class="author-block">
-              <a href="https://www.danbgoldman.com" target="_blank">Dan B Goldman</a><sup>2</sup>,
-            </span>
-            <span class="author-block">
-              <a href="https://homes.cs.washington.edu/~seitz/" target="_blank">Steven M. Seitz</a><sup>1,2</sup>,
-            </span>
-            <span class="author-block">
-              <a href="http://www.ricardomartinbrualla.com" target="_blank">Ricardo Martin-Brualla</a><sup>2</sup>
+              <a href="https://www.ilievski.info/" target="_blank">Filip Ilievski </a><img src="./static/images/vu_logo.png" style="height: 1em; vertical-align: middle;">
             </span>
           </div>
-
           <div class="is-size-5 publication-authors">
-            <span class="author-block"><sup>1</sup>University of Washington,</span>
-            <span class="author-block"><sup>2</sup>Google Research</span>
+            <span class="author-block"><img src="./static/images/usc_logo.png" style="height: 1em; vertical-align: middle;"> University of Southern California, USA</span> <br>
+            <span class="author-block"><img src="./static/images/vu_logo.png" style="height: 1em; vertical-align: middle;"> Vrije Universiteit Amsterdam, The Netherlands</span>
           </div>
 
+
           <div class="column has-text-centered">
             <div class="publication-links">
               <!-- PDF Link. -->
@@ -83,7 +75,7 @@
                 </a>
               </span>
               <!-- Video Link. -->
-              <span class="link-block">
+              <!-- <span class="link-block">
                 <a href="https://www.youtube.com/watch?v=MrKrnHhk8IA" target="_blank"
                    class="external-link button is-normal is-rounded is-dark">
                   <span class="icon">
@@ -91,10 +83,10 @@
                   </span>
                   <span>Video</span>
                 </a>
-              </span>
+              </span> -->
               <!-- Code Link. -->
               <span class="link-block">
-                <a href="https://github.com/google/nerfies" target="_blank"
+                <a href="https://github.com/saccharomycetes/mllms_know" target="_blank"
                    class="external-link button is-normal is-rounded is-dark">
                   <span class="icon">
                       <i class="fab fa-github"></i>
@@ -103,7 +95,7 @@
                   </a>
               </span>
               <!-- Dataset Link. -->
-              <span class="link-block">
+              <!-- <span class="link-block">
                 <a href="https://github.com/google/nerfies/releases/tag/0.1" target="_blank"
                    class="external-link button is-normal is-rounded is-dark">
                   <span class="icon">
@@ -111,83 +103,33 @@
                   </span>
                   <span>Data</span>
                   </a>
-            </div>
+            </div> -->
 
           </div>
+
+          <div class="column has-text-centered" style="margin: 1.5rem 0; padding: 0.75rem; background: linear-gradient(45deg, #4a90e2, #50e3c2); color: white; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.1); text-align: center;">
+            Accepted at ICLR 2025
+          </div>
+
+
         </div>
       </div>
     </div>
   </div>
 </section>
 
-<section class="hero teaser">
-  <div class="container is-max-desktop">
-    <div class="hero-body">
-      <video id="teaser" autoplay muted loop playsinline height="100%">
-        <source src="./static/videos/teaser.mp4"
-                type="video/mp4">
-      </video>
-      <h2 class="subtitle has-text-centered">
-        <span class="dnerf">Nerfies</span> turns selfie videos from your phone into
-        free-viewpoint
-        portraits.
-      </h2>
-    </div>
-  </div>
-</section>
 
 
-<section class="hero is-light is-small">
-  <div class="hero-body">
-    <div class="container">
-      <div id="results-carousel" class="carousel results-carousel">
-        <div class="item item-steve">
-          <video poster="" id="steve" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/steve.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <div class="item item-chair-tp">
-          <video poster="" id="chair-tp" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/chair-tp.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <div class="item item-shiba">
-          <video poster="" id="shiba" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/shiba.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <div class="item item-fullbody">
-          <video poster="" id="fullbody" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/fullbody.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <div class="item item-blueshirt">
-          <video poster="" id="blueshirt" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/blueshirt.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <div class="item item-mask">
-          <video poster="" id="mask" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/mask.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <div class="item item-coffee">
-          <video poster="" id="coffee" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/coffee.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <div class="item item-toby">
-          <video poster="" id="toby" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/toby2.mp4"
-                    type="video/mp4">
-          </video>
+<section class="section">
+  <div class="container is-max-desktop">
+    <!-- Paper poster. -->
+    <div class="columns is-centered has-text-centered">
+      <div class="column is-four-fifths">
+        <div class="publication-image">
+          <img src="./static/images/motivation_case.jpg" alt="Paper poster image">
+          <p class="caption">
+            Examples of MLLMs knowing where to look despite answering incorrectly. The right panel in each example displays relative attention to image of one layer in the MLLM.
+          </p>
         </div>
       </div>
     </div>
@@ -203,187 +145,155 @@
         <h2 class="title is-3">Abstract</h2>
         <div class="content has-text-justified">
           <p>
-            We present the first method capable of photorealistically reconstructing a non-rigidly
-            deforming scene using photos/videos captured casually from mobile phones.
-          </p>
-          <p>
-            Our approach augments neural radiance fields
-            (NeRF) by optimizing an
-            additional continuous volumetric deformation field that warps each observed point into a
-            canonical 5D NeRF.
-            We observe that these NeRF-like deformation fields are prone to local minima, and
-            propose a coarse-to-fine optimization method for coordinate-based models that allows for
-            more robust optimization.
-            By adapting principles from geometry processing and physical simulation to NeRF-like
-            models, we propose an elastic regularization of the deformation field that further
-            improves robustness.
-          </p>
-          <p>
-            We show that <span class="dnerf">Nerfies</span> can turn casually captured selfie
-            photos/videos into deformable NeRF
-            models that allow for photorealistic renderings of the subject from arbitrary
-            viewpoints, which we dub <i>"nerfies"</i>. We evaluate our method by collecting data
-            using a
-            rig with two mobile phones that take time-synchronized photos, yielding train/validation
-            images of the same pose at different viewpoints. We show that our method faithfully
-            reconstructs non-rigidly deforming scenes and reproduces unseen views with high
-            fidelity.
+            Multimodal Large Language Models (MLLMs) have experience rapid progress in recent years. Given their potential integration into many critical applications, it is important to understand the limitations of their perception ability. In this work, we study whether MLLMs can perceive small detailed visual information as well as large ones in images. In particular, we observe that their accuracy in answering visual questions is very sensitive to the size of the visual subject of the question. We further show that this effect is causal by observing that human visual cropping can significantly mitigate this sensitivity. Next, we study the attention patterns of MLLMs when answering visual questions, and intriguingly find that they consistently know where to look, even when they provide the wrong answer. Based on these findings, we then construct automatic visual cropping methods that leverage the internal knowledge of any MLLM itself, in the form of attention and gradient maps, to help it better perceive the small visual subject of any question. We study our proposed methods on two popular MLLMs and seven multimodal benchmarks, and show that they can significantly improve MLLMs' accuracy <b>without requiring any training</b>. Our findings suggest that MLLMs should be used with caution in detail-sensitive applications, and that visual cropping with model's own knowledge is a promising direction to improve their performance.
           </p>
         </div>
       </div>
     </div>
     <!--/ Abstract. -->
+</section>
 
-    <!-- Paper video. -->
+<section class="section">
+  <div class="container is-max-desktop">
+    <!-- Paper poster. -->
     <div class="columns is-centered has-text-centered">
       <div class="column is-four-fifths">
-        <h2 class="title is-3">Video</h2>
-        <div class="publication-video">
-          <iframe src="https://www.youtube.com/embed/MrKrnHhk8IA?rel=0&amp;showinfo=0"
-                  frameborder="0" allow="autoplay; encrypted-media" allowfullscreen></iframe>
+        <h3 class="title is-3" style="text-align: left">Automatic Visual Cropping</h3>
+        <div class="publication-image">
+          <img src="./static/images/vicrop_methods.jpg" alt="Paper poster image">
+          <p class="caption">
+            Illustration of the proposed visual cropping approach applied to two MLLMs.
+          </p>
         </div>
       </div>
     </div>
-    <!--/ Paper video. -->
   </div>
 </section>
 
-
 <section class="section">
   <div class="container is-max-desktop">
-
-    <div class="columns is-centered">
-
-      <!-- Visual Effects. -->
-      <div class="column">
-        <div class="content">
-          <h2 class="title is-3">Visual Effects</h2>
-          <p>
-            Using <i>nerfies</i> you can create fun visual effects. This Dolly zoom effect
-            would be impossible without nerfies since it would require going through a wall.
+    <!-- Paper poster. -->
+    <div class="columns is-centered has-text-centered">
+      <div class="column is-four-fifths">
+        <h3 class="title is-3" style="text-align: left">Visual Cropping Methods Analysis</h3>
+        <div class="publication-image">
+          <img src="./static/images/method_case.jpg" alt="Paper poster image">
+          <p class="caption">
+            Examples of rel-att helping MLLMs correct their mistakes (<i>cyan-colored bounding box shows cropped region by rel-att; zoom-in insets are displayed for better readability</i>).
           </p>
-          <video id="dollyzoom" autoplay controls muted loop playsinline height="100%">
-            <source src="./static/videos/dollyzoom-stacked.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-      </div>
-      <!--/ Visual Effects. -->
-
-      <!-- Matting. -->
-      <div class="column">
-        <h2 class="title is-3">Matting</h2>
-        <div class="columns is-centered">
-          <div class="column content">
-            <p>
-              As a byproduct of our method, we can also solve the matting problem by ignoring
-              samples that fall outside of a bounding box during rendering.
-            </p>
-            <video id="matting-video" controls playsinline height="100%">
-              <source src="./static/videos/matting.mp4"
-                      type="video/mp4">
-            </video>
-          </div>
-
         </div>
       </div>
     </div>
-    <!--/ Matting. -->
-
-    <!-- Animation. -->
-    <div class="columns is-centered">
-      <div class="column is-full-width">
-        <h2 class="title is-3">Animation</h2>
-
-        <!-- Interpolating. -->
-        <h3 class="title is-4">Interpolating states</h3>
-        <div class="content has-text-justified">
-          <p>
-            We can also animate the scene by interpolating the deformation latent codes of two input
-            frames. Use the slider here to linearly interpolate between the left frame and the right
-            frame.
-          </p>
-        </div>
-        <div class="columns is-vcentered interpolation-panel">
-          <div class="column is-3 has-text-centered">
-            <img src="./static/images/interpolate_start.jpg"
-                 class="interpolation-image"
-                 alt="Interpolate start reference image."/>
-            <p>Start Frame</p>
-          </div>
-          <div class="column interpolation-video-column">
-            <div id="interpolation-image-wrapper">
-              Loading...
-            </div>
-            <input class="slider is-fullwidth is-large is-info"
-                   id="interpolation-slider"
-                   step="1" min="0" max="100" value="0" type="range">
-          </div>
-          <div class="column is-3 has-text-centered">
-            <img src="./static/images/interpolate_end.jpg"
-                 class="interpolation-image"
-                 alt="Interpolation end reference image."/>
-            <p class="is-bold">End Frame</p>
-          </div>
-        </div>
-        <br/>
-        <!--/ Interpolating. -->
-
-        <!-- Re-rendering. -->
-        <h3 class="title is-4">Re-rendering the input video</h3>
-        <div class="content has-text-justified">
-          <p>
-            Using <span class="dnerf">Nerfies</span>, you can re-render a video from a novel
-            viewpoint such as a stabilized camera by playing back the training deformations.
-          </p>
-        </div>
-        <div class="content has-text-centered">
-          <video id="replay-video"
-                 controls
-                 muted
-                 preload
-                 playsinline
-                 width="75%">
-            <source src="./static/videos/replay.mp4"
-                    type="video/mp4">
-          </video>
-        </div>
-        <!--/ Re-rendering. -->
+  </div>
+</section>
 
-      </div>
-    </div>
-    <!--/ Animation. -->
+<table class="table is-bordered is-striped is-narrow is-hoverable" style="margin: 0 auto; width: auto; font-size: 0.9em;">
+  <caption style="caption-side: top; margin-bottom: 1em; font-weight: bold; color: #363636;">Accuracy of the proposed ViCrop methods on visual question answering benchmarks.</caption>
+  <thead>
+    <tr>
+      <th rowspan="2" colspan="2" style="vertical-align: middle; background-color: #f5f5f5;">Model</th>
+      <th colspan="4" style="text-align: center; background-color: #f0f8ff;">Smaller Visual Concepts</th>
+      <th colspan="3" style="text-align: center; background-color: #fff0f5;">Larger Visual Concepts</th>
+    </tr>
+    <tr>
+      <th style="background-color: #f0f8ff;">TextVQA†</th>
+      <th style="background-color: #f0f8ff;">V*</th>
+      <th style="background-color: #f0f8ff;">POPE</th>
+      <th style="background-color: #f0f8ff;">DocVQA</th>
+      <th style="background-color: #fff0f5;">AOKVQA</th>
+      <th style="background-color: #fff0f5;">GQA</th>
+      <th style="background-color: #fff0f5;">VQAv2</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td rowspan="4" style="font-weight: bold; background-color: #f5f5f5;">LLAVA-1.5</td>
+      <td style="font-weight: bold;">no cropping</td>
+      <td>47.80</td>
+      <td>42.41</td>
+      <td>85.27</td>
+      <td>15.97</td>
+      <td>59.01</td>
+      <td>60.48</td>
+      <td>75.57</td>
+    </tr>
+    <tr>
+      <td style="font-weight: bold;">rel-att</td>
+      <td>55.17</td>
+      <td style="font-weight: bold; color: #000000;">62.30</td>
+      <td style="font-weight: bold; color: #000000;">87.25</td>
+      <td>19.63</td>
+      <td style="font-weight: bold; color: #000000;">60.66</td>
+      <td>60.97</td>
+      <td style="font-weight: bold; color: #000000;">76.51</td>
+    </tr>
+    <tr>
+      <td style="font-weight: bold;">grad-att</td>
+      <td style="font-weight: bold; color: #000000;">56.06</td>
+      <td>57.07</td>
+      <td>87.03</td>
+      <td style="font-weight: bold; color: #000000;">19.84</td>
+      <td>59.94</td>
+      <td style="font-weight: bold; color: #000000;">60.98</td>
+      <td>76.06</td>
+    </tr>
+    <tr>
+      <td style="font-weight: bold;">pure-grad</td>
+      <td>51.67</td>
+      <td>46.07</td>
+      <td>86.06</td>
+      <td>17.70</td>
+      <td>59.92</td>
+      <td>60.54</td>
+      <td>75.94</td>
+    </tr>
+    <tr>
+      <td rowspan="4" style="font-weight: bold; background-color: #f5f5f5;">InstructBLIP</td>
+      <td style="font-weight: bold;">no cropping</td>
+      <td>33.48</td>
+      <td>35.60</td>
+      <td>84.89</td>
+      <td>9.20</td>
+      <td>60.06</td>
+      <td>49.41</td>
+      <td>76.25</td>
+    </tr>
+    <tr>
+      <td style="font-weight: bold;">rel-att</td>
+      <td>45.44</td>
+      <td style="font-weight: bold; color: #000000;">42.41</td>
+      <td>86.64</td>
+      <td>9.95</td>
+      <td>61.28</td>
+      <td>49.75</td>
+      <td style="font-weight: bold; color: #000000;">76.84</td>
+    </tr>
+    <tr>
+      <td style="font-weight: bold;">grad-att</td>
+      <td style="font-weight: bold; color: #000000;">45.71</td>
+      <td>37.70</td>
+      <td style="font-weight: bold; color: #000000;">86.99</td>
+      <td style="font-weight: bold; color: #000000;">10.81</td>
+      <td style="font-weight: bold; color: #000000;">61.77</td>
+      <td style="font-weight: bold; color: #000000;">50.33</td>
+      <td>76.08</td>
+    </tr>
+    <tr>
+      <td style="font-weight: bold;">pure-grad</td>
+      <td>42.23</td>
+      <td>37.17</td>
+      <td>86.84</td>
+      <td>8.99</td>
+      <td>61.60</td>
+      <td>50.08</td>
+      <td>76.71</td>
+    </tr>
+  </tbody>
+</table>
 
 
-    <!-- Concurrent Work. -->
-    <div class="columns is-centered">
-      <div class="column is-full-width">
-        <h2 class="title is-3">Related Links</h2>
 
-        <div class="content has-text-justified">
-          <p>
-            There's a lot of excellent work that was introduced around the same time as ours.
-          </p>
-          <p>
-            <a href="https://arxiv.org/abs/2104.09125" target="_blank">Progressive Encoding for Neural Optimization</a> introduces an idea similar to our windowed position encoding for coarse-to-fine optimization.
-          </p>
-          <p>
-            <a href="https://www.albertpumarola.com/research/D-NeRF/index.html" target="_blank">D-NeRF</a> and <a href="https://gvv.mpi-inf.mpg.de/projects/nonrigid_nerf/" target="_blank">NR-NeRF</a>
-            both use deformation fields to model non-rigid scenes.
-          </p>
-          <p>
-            Some works model videos with a NeRF by directly modulating the density, such as <a href="https://video-nerf.github.io/" target="_blank">Video-NeRF</a>, <a href="https://www.cs.cornell.edu/~zl548/NSFF/" target="_blank">NSFF</a>, and <a href="https://neural-3d-video.github.io/" target="_blank">DyNeRF</a>
-          </p>
-          <p>
-            There are probably many more by the time you are reading this. Check out <a href="https://dellaert.github.io/NeRF/" target="_blank">Frank Dellart's survey on recent NeRF papers</a>, and <a href="https://github.com/yenchenlin/awesome-NeRF" target="_blank">Yen-Chen Lin's curated list of NeRF papers</a>.
-          </p>
-        </div>
-      </div>
-    </div>
-    <!--/ Concurrent Work. -->
 
-  </div>
-</section>
 
 
 <section class="section" id="BibTeX">