clean up

app.py

@@ -8,7 +8,6 @@ import torch
 import matplotlib.pyplot as plt
 from attn_helper import VITAttentionGradRollout, overlay_attn
 import vc_models
-#import eaif_models
 import torchvision
 
 
@@ -64,7 +63,7 @@ def download_bin(model):
         os.rename(model_bin, bin_path)
 
 
-def run_attn(input_img, model="vc1-large",fusion="min",slider=0):
+def run_attn(input_img, model="vc1-large"):
     download_bin(model)
     model, embedding_dim, transform, metadata = get_model(model)
     if input_img.shape[0] != 3:
@@ -76,32 +75,22 @@ def run_attn(input_img, model="vc1-large",fusion="min",slider=0):
     input_img = resize_transform(input_img)
     x = transform(input_img)
 
-    attention_rollout = VITAttentionGradRollout(model,head_fusion=fusion,discard_ratio=slider)
+    attention_rollout = VITAttentionGradRollout(model,head_fusion="max",discard_ratio=0.89)
     
     y = model(x)
     mask = attention_rollout.get_attn_mask()
     attn_img = overlay_attn(input_img[0].permute(1,2,0),mask)
-    
-    fig = plt.figure()
-    ax = fig.subplots()
-    print(y.shape)
-    im = ax.matshow(y.detach().numpy().reshape(16,-1))
-    plt.colorbar(im)
-
-    return attn_img, fig
+    return attn_img
 
 model_type = gr.Dropdown(
             ["vc1-base", "vc1-large"], label="Model Size", value="vc1-large")
 input_img = gr.Image(shape=(250,250))
-input_button = gr.Radio(["min", "max", "mean"], value="min",label="Attention Head Fusion", info="How to combine the last layer attention across all 12 heads of the transformer.")
 output_img = gr.Image(shape=(250,250))
-output_plot = gr.Plot()
-css = "#component-3, .input-image, .image-preview {height: 240px !important}"
-slider = gr.Slider(0, 1)
+css = "#component-2, .input-image, .image-preview {height: 240px !important}"
 markdown ="This is a demo for the Visual Cortex models. When passed an image input, it displays the attention of the last layer of the transformer.\n \
          The user can decide how the attention heads will be combined. \
          Along with the attention heatmap, it also displays the embedding values reshaped to a 16x48 or 16x64 grid."
 demo = gr.Interface(fn=run_attn, title="Visual Cortex Large Model", description=markdown,
-                    examples=[[os.path.join('./imgs',x),None,None,None]for x in os.listdir(os.path.join(os.getcwd(),'imgs')) if 'jpg' in x],
-                    inputs=[input_img,model_type,input_button,slider],outputs=[output_img,output_plot],css=css)
+                    examples=[[os.path.join('./imgs',x),None]for x in os.listdir(os.path.join(os.getcwd(),'imgs')) if 'jpg' in x],
+                    inputs=[input_img,model_type],outputs=output_img,css=css)
 demo.launch()

attn_helper.py

@@ -18,7 +18,6 @@ def overlay_attn(original_image,mask):
     # Apply colormap to mask
     cmap = cv2.applyColorMap(np.uint8(255 * mask), colormap_attn)
 
-    print(cmap.shape)
     # Blend mask and original image
     # grayscale_img =  cv2.cvtColor(np.uint8(original_image), cv2.COLOR_RGB2GRAY)
     # grayscale_img = cv2.cvtColor(grayscale_img, cv2.COLOR_GRAY2RGB)
@@ -45,7 +44,6 @@ class VITAttentionGradRollout:
         self.model = model
         self.head_fusion = head_fusion
         self.discard_ratio = discard_ratio
-        print(list(model.blocks.children()))
         
         self.attentions = {}
         for idx, module in enumerate(list(model.blocks.children())):