huzey commited on
Commit
62031d2
1 Parent(s): da6c997
Files changed (1) hide show
  1. app.py +55 -53
app.py CHANGED
@@ -948,6 +948,60 @@ with demo:
948
  from draft_gradio_app_text import make_demo
949
  make_demo()
950
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
951
  with gr.Tab('Compare Models'):
952
  def add_one_model(i_model=1):
953
  with gr.Column(scale=5, min_width=200) as col:
@@ -1015,59 +1069,7 @@ with demo:
1015
  buttons[-1].click(fn=lambda x: gr.update(visible=True), outputs=rows[-1])
1016
  buttons[-1].click(fn=lambda x: gr.update(visible=False), outputs=buttons[-1])
1017
 
1018
- with gr.Tab('Compare (Aligned)'):
1019
- gr.Markdown('This page reproduce the results from the paper [AlignedCut](https://arxiv.org/abs/2406.18344)')
1020
- gr.Markdown('---')
1021
- gr.Markdown('**Features are aligned across models and layers.** A linear alignment transform is trained for each model/layer, learning signal comes from 1) fMRI brain activation and 2) segmentation preserving eigen-constraints.')
1022
- gr.Markdown('NCUT is computed on the concatenated graph of all models, layers, and images. Color is **aligned** across all models and layers.')
1023
- gr.Markdown('')
1024
- gr.Markdown("To see a good pattern, you will need to load 100~1000 images. 100 images need 10sec for RTX4090. Running out of HuggingFace GPU Quota? Try [Demo](https://ncut-pytorch.readthedocs.io/en/latest/demo/) hosted at UPenn")
1025
- gr.Markdown('---')
1026
- with gr.Row():
1027
- with gr.Column(scale=5, min_width=200):
1028
- input_gallery, submit_button, clear_images_button = make_input_images_section()
1029
-
1030
- dataset_dropdown, num_images_slider, random_seed_slider, load_images_button = make_dataset_images_section(advanced=True)
1031
- num_images_slider.value = 100
1032
-
1033
- with gr.Column(scale=5, min_width=200):
1034
- gr.Markdown('Model: CLIP(ViT-B-16/openai), DiNOv2reg(dinov2_vitb14_reg), MAE(vit_base)')
1035
- gr.Markdown('Layer type: attention output (attn), without sum of residual')
1036
- [
1037
- model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
1038
- affinity_focal_gamma_slider, num_sample_ncut_slider, knn_ncut_slider,
1039
- embedding_method_dropdown, num_sample_tsne_slider, knn_tsne_slider,
1040
- perplexity_slider, n_neighbors_slider, min_dist_slider,
1041
- sampling_method_dropdown
1042
- ] = make_parameters_section()
1043
- model_dropdown.value = "AlignedThreeModelAttnNodes"
1044
- model_dropdown.visible = False
1045
- layer_slider.visible = False
1046
- node_type_dropdown.visible = False
1047
- # logging text box
1048
- logging_text = gr.Textbox("Logging information", label="Logging", elem_id="logging", type="text", placeholder="Logging information")
1049
-
1050
- galleries = []
1051
- for i_model, model_name in enumerate(["CLIP", "DINO", "MAE"]):
1052
- with gr.Row():
1053
- for i_layer in range(1, 13):
1054
- with gr.Column(scale=5, min_width=200):
1055
- gr.Markdown(f'### {model_name} Layer {i_layer}')
1056
- output_gallery = gr.Gallery(value=[], label="NCUT Embedding", show_label=False, elem_id="ncut", columns=[3], rows=[1], object_fit="contain", height="auto")
1057
- galleries.append(output_gallery)
1058
-
1059
-
1060
- clear_images_button.click(lambda x: [] * (len(galleries) + 1), outputs=[input_gallery] + galleries)
1061
- submit_button.click(
1062
- run_fn,
1063
- inputs=[
1064
- input_gallery, model_dropdown, layer_slider, num_eig_slider, node_type_dropdown,
1065
- affinity_focal_gamma_slider, num_sample_ncut_slider, knn_ncut_slider,
1066
- embedding_method_dropdown, num_sample_tsne_slider, knn_tsne_slider,
1067
- perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown
1068
- ],
1069
- outputs=galleries + [logging_text],
1070
- )
1071
 
1072
  with gr.Row():
1073
  with gr.Column():
 
948
  from draft_gradio_app_text import make_demo
949
  make_demo()
950
 
951
+ with gr.Tab('Model Aligned'):
952
+ gr.Markdown('This page reproduce the results from the paper [AlignedCut](https://arxiv.org/abs/2406.18344)')
953
+ gr.Markdown('---')
954
+ gr.Markdown('**Features are aligned across models and layers.** A linear alignment transform is trained for each model/layer, learning signal comes from 1) fMRI brain activation and 2) segmentation preserving eigen-constraints.')
955
+ gr.Markdown('NCUT is computed on the concatenated graph of all models, layers, and images. Color is **aligned** across all models and layers.')
956
+ gr.Markdown('')
957
+ gr.Markdown("To see a good pattern, you will need to load 100~1000 images. 100 images need 10sec for RTX4090. Running out of HuggingFace GPU Quota? Try [Demo](https://ncut-pytorch.readthedocs.io/en/latest/demo/) hosted at UPenn")
958
+ gr.Markdown('---')
959
+ with gr.Row():
960
+ with gr.Column(scale=5, min_width=200):
961
+ input_gallery, submit_button, clear_images_button = make_input_images_section()
962
+
963
+ dataset_dropdown, num_images_slider, random_seed_slider, load_images_button = make_dataset_images_section(advanced=True)
964
+ num_images_slider.value = 100
965
+
966
+ with gr.Column(scale=5, min_width=200):
967
+ gr.Markdown('Model: CLIP(ViT-B-16/openai), DiNOv2reg(dinov2_vitb14_reg), MAE(vit_base)')
968
+ gr.Markdown('Layer type: attention output (attn), without sum of residual')
969
+ [
970
+ model_dropdown, layer_slider, node_type_dropdown, num_eig_slider,
971
+ affinity_focal_gamma_slider, num_sample_ncut_slider, knn_ncut_slider,
972
+ embedding_method_dropdown, num_sample_tsne_slider, knn_tsne_slider,
973
+ perplexity_slider, n_neighbors_slider, min_dist_slider,
974
+ sampling_method_dropdown
975
+ ] = make_parameters_section()
976
+ model_dropdown.value = "AlignedThreeModelAttnNodes"
977
+ model_dropdown.visible = False
978
+ layer_slider.visible = False
979
+ node_type_dropdown.visible = False
980
+ # logging text box
981
+ logging_text = gr.Textbox("Logging information", label="Logging", elem_id="logging", type="text", placeholder="Logging information")
982
+
983
+ galleries = []
984
+ for i_model, model_name in enumerate(["CLIP", "DINO", "MAE"]):
985
+ with gr.Row():
986
+ for i_layer in range(1, 13):
987
+ with gr.Column(scale=5, min_width=200):
988
+ gr.Markdown(f'### {model_name} Layer {i_layer}')
989
+ output_gallery = gr.Gallery(value=[], label="NCUT Embedding", show_label=False, elem_id="ncut", columns=[3], rows=[1], object_fit="contain", height="auto")
990
+ galleries.append(output_gallery)
991
+
992
+
993
+ clear_images_button.click(lambda x: [] * (len(galleries) + 1), outputs=[input_gallery] + galleries)
994
+ submit_button.click(
995
+ run_fn,
996
+ inputs=[
997
+ input_gallery, model_dropdown, layer_slider, num_eig_slider, node_type_dropdown,
998
+ affinity_focal_gamma_slider, num_sample_ncut_slider, knn_ncut_slider,
999
+ embedding_method_dropdown, num_sample_tsne_slider, knn_tsne_slider,
1000
+ perplexity_slider, n_neighbors_slider, min_dist_slider, sampling_method_dropdown
1001
+ ],
1002
+ outputs=galleries + [logging_text],
1003
+ )
1004
+
1005
  with gr.Tab('Compare Models'):
1006
  def add_one_model(i_model=1):
1007
  with gr.Column(scale=5, min_width=200) as col:
 
1069
  buttons[-1].click(fn=lambda x: gr.update(visible=True), outputs=rows[-1])
1070
  buttons[-1].click(fn=lambda x: gr.update(visible=False), outputs=buttons[-1])
1071
 
1072
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1073
 
1074
  with gr.Row():
1075
  with gr.Column():