Upload 25 files

cs772_proj/app.py

@@ -0,0 +1,372 @@
+import gradio as gr
+
+import transformers 
+import torch
+#import neptune
+#from knockknock import slack_sender
+from transformers import *
+#import glob 
+from transformers import BertTokenizer
+from transformers import BertForSequenceClassification, AdamW, BertConfig
+import random
+import pandas as pd
+from transformers import BertTokenizer
+#from Models.utils import masked_cross_entropy,fix_the_random,format_time,save_normal_model,save_bert_model
+from sklearn.metrics import accuracy_score,f1_score
+from tqdm import tqdm
+'''from TensorDataset.datsetSplitter import createDatasetSplit
+from TensorDataset.dataLoader import combine_features
+from Preprocess.dataCollect import collect_data,set_name'''
+from sklearn.metrics import accuracy_score,f1_score,roc_auc_score,recall_score,precision_score
+import matplotlib.pyplot as plt
+import time
+import os
+from transformers import BertTokenizer
+#import GPUtil
+from sklearn.utils import class_weight
+#import json
+#from Models.bertModels import *
+#from Models.otherModels import *
+import sys
+#import time
+#from waiting import wait
+from sklearn.preprocessing import LabelEncoder
+import numpy as np
+#import threading
+#import argparse
+#import ast
+
+#from manual_training_inference import select_model
+#from Models.utils import save_normal_model,save_bert_model,load_model
+#from Models.utils import return_params
+from transformers import DistilBertTokenizer
+
+
+#from TensorDataset.dataLoader import custom_att_masks
+#from keras.preprocessing.sequence import pad_sequences
+
+#import seaborn as sns 
+import matplotlib.pyplot as plt 
+import numpy as np
+import PIL.Image as Image
+from torch import nn
+
+from pyvene import embed_to_distrib, top_vals, format_token
+from pyvene import (
+    IntervenableModel,
+    VanillaIntervention, Intervention,
+    RepresentationConfig,
+    IntervenableConfig,
+    ConstantSourceIntervention,
+    LocalistRepresentationIntervention
+)
+from pyvene import create_gpt2
+#%config InlineBackend.figure_formats = ['svg']
+from plotnine import (
+    ggplot,
+    geom_tile,
+    aes,
+    facet_wrap,
+    theme,
+    element_text,
+    geom_bar,
+    geom_hline,
+    scale_y_log10,
+    xlab, ylab, ylim,
+    scale_y_discrete, scale_y_continuous, ggsave
+)
+from plotnine.scales import scale_y_reverse, scale_fill_cmap
+from tqdm import tqdm
+global device
+device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+def create_bert(cache_dir=None):
+    """Creates a GPT2 model, config, and tokenizer from the given name and revision"""
+    from transformers import BertConfig
+
+    config = BertConfig.from_pretrained("./cs77_proj/bert_base/checkpoint-3848/config.json")
+    tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+    gpt = AutoModelForSequenceClassification.from_pretrained("./cs77_proj/bert_base/checkpoint-3848", config=config, cache_dir=cache_dir)
+    print("loaded model")
+    return config, tokenizer, gpt
+def interpret(text,label):
+            titles={
+            "block_output": "single restored layer in BERT",
+            "mlp_activation": "center of interval of 5 patched mlp layer",
+            "attention_output": "center of interval of 5 patched attn layer"
+        }
+
+            colors={
+            "block_output": "Purples",
+            "mlp_activation": "Greens",
+            "attention_output": "Reds"
+        } 
+         
+            device = "cuda:0" if torch.cuda.is_available() else "cpu"
+            #config, tokenizer, gpt =  pv.create_llama(name="sharpbai/alpaca-7b-merged")
+            config, tokenizer, gpt = create_bert()
+            #config, tokenizer, gpt = create_gpt2(name="gpt2-xl")
+
+            gpt.to(device)
+
+            base = text
+            inputs = [
+                tokenizer(base, return_tensors="pt").to(device),
+            ]
+            #print(base)
+            base_token = tokenizer.convert_ids_to_tokens(inputs[0]['input_ids'][0])
+            res = gpt(**inputs[0])
+            probabilities = nn.functional.softmax(res[0], dim=-1)
+            if label=="hate":
+                 l = 0
+            elif label=="normal":
+                 l=1
+            else:l=2
+            #print(probabilities)
+            #print(res[0][0][0].item())
+            #print(res)
+            #distrib = embed_to_distrib(gpt, res.last_hidden_state, logits=False)
+            #top_vals(tokenizer, distrib[0][-1], n=20)
+            base = tokenizer(text, return_tensors="pt").to(device)
+            config = corrupted_config(type(gpt))
+            intervenable = IntervenableModel(config, gpt)
+            _, counterfactual_outputs = intervenable(
+                base, unit_locations={"base": ([[[0,1,2,3]]])}
+            )
+            #probabilities = nn.functional.softmax(counterfactual_outputs[0], dim=-1)
+            #print(probabilities)
+            for stream in ["block_output", "mlp_activation", "attention_output"]:
+                data = []
+                for layer_i in tqdm(range(gpt.config.num_hidden_layers)):
+                    for pos_i in range(len(base_token)):
+                        config = restore_corrupted_with_interval_config(
+                            layer_i, stream, 
+                            window=1 if stream == "block_output" else 5
+                        )
+                        
+                        n_restores = len(config.representations) - 1
+                        intervenable = IntervenableModel(config, gpt)
+                        _, counterfactual_outputs = intervenable(
+                            base,
+                            [None] + [base]*n_restores,
+                            {
+                                "sources->base": (
+                                    [None] + [[[pos_i]]]*n_restores,
+                                    [[[0,1,2,3]]] + [[[pos_i]]]*n_restores,
+                                )
+                            },
+                        )
+                        #distrib = embed_to_distrib(
+                            #gpt, counterfactual_outputs.last_hidden_state, logits=False
+                        #)
+                        #prob = distrib[0][-1][token].detach().cpu().item()
+                        logits = counterfactual_outputs[0]
+                        probabilities = nn.functional.softmax(logits, dim=-1)
+                        prob_offense = probabilities[0][l].item()
+                        data.append({"layer": layer_i, "pos": pos_i, "prob": prob_offense})
+                df = pd.DataFrame(data)
+                df.to_csv(f"./cs77_proj/tutorial_data/pyvene_rome_{stream}.csv")
+            for stream in ["block_output", "mlp_activation", "attention_output"]:
+                    df = pd.read_csv(f"./cs77_proj/tutorial_data/pyvene_rome_{stream}.csv")
+                    df["layer"] = df["layer"].astype(int)
+                    df["pos"] = df["pos"].astype(int)
+                    prob_type = "p"+"("+label+")"
+                    df[prob_type] = df["prob"].astype(float)
+                    #custom_labels = ["imagine*","the*", "riots*", "if", "people", "actually", "got" ,"food" ,"boxes" ,"instead", "of" ,"ebt", "cards", "every", "ghetto", "in", "america", "would" ,"look", "like", "ferguson"]
+                    custom_labels = base_token #["what*", "sort*", "of*", "white*","man" ,"or", "woman", "would", "vote", "for", "this", "nigger"]
+                    #custom_labels = ["no*", "liberal*","congratulated*", "hindu*", "refugees", "post", "cab", "because", "they", "hate", "hindus"]
+                    breaks = list(range(len(custom_labels)))#[0, 1, 2, 3, 4, 5, 6,7,8,9,10,11]
+
+
+                    plot = (
+                        ggplot(df, aes(x="layer", y="pos"))    
+
+                        + geom_tile(aes(fill=prob_type))
+                        + scale_fill_cmap(colors[stream]) + xlab(titles[stream])
+                        + scale_y_reverse(
+                            limits = (-0.5, len(custom_labels)), 
+                            breaks=breaks, labels=custom_labels) 
+                        + theme(figure_size=(6,9)) + ylab("") 
+                        + theme(axis_text_y  = element_text(angle = 90, hjust = 1))
+                    )
+                    ggsave(
+                        plot, filename=f"./cs77_proj/tutorial_data/pyvene_rome_{stream}.png", dpi=200
+                    )
+                    if stream == "mlp_activation":
+                        mlp_img_path = f"./cs77_proj/tutorial_data/pyvene_rome_{stream}.png"
+                    elif stream=="block_output":
+                        bo_path = f"./cs77_proj/tutorial_data/pyvene_rome_{stream}.png"
+                    else:attention_path = f"./cs77_proj/tutorial_data/pyvene_rome_{stream}.png"
+            return mlp_img_path,bo_path,attention_path
+
+def restore_corrupted_with_interval_config(
+    layer, stream="mlp_activation", window=5, num_layers=12):
+    start = max(0, layer - window // 2)
+    end = min(num_layers, layer - (-window // 2))
+    config = IntervenableConfig(
+        representations=[
+            RepresentationConfig(
+                0,       # layer
+                "block_input",  # intervention type
+            ),
+        ] + [
+            RepresentationConfig(
+                i,       # layer
+                stream,  # intervention type
+        ) for i in range(start, end)],
+        intervention_types=\
+            [NoiseIntervention]+[VanillaIntervention]*(end-start)
+    )
+    return config
+
+class NoiseIntervention(ConstantSourceIntervention, LocalistRepresentationIntervention):
+    def __init__(self, embed_dim, **kwargs):
+        super().__init__()
+        self.interchange_dim = embed_dim
+        rs = np.random.RandomState(1)
+        prng = lambda *shape: rs.randn(*shape)
+        self.noise = torch.from_numpy(
+            prng(1, 4, embed_dim)).to(device)
+        self.noise_level = 0.7462981581687927 #0.3462981581687927
+
+    def forward(self, base, source=None, subspaces=None):
+        base[..., : self.interchange_dim] += self.noise * self.noise_level
+        return base
+
+    def __str__(self):
+        return f"NoiseIntervention(embed_dim={self.embed_dim})"
+
+
+def corrupted_config(model_type):
+    config = IntervenableConfig(
+        model_type=model_type,
+        representations=[
+            RepresentationConfig(
+                0,              # layer
+                "block_input",  # intervention type
+            ),
+        ],
+        intervention_types=NoiseIntervention,
+    )
+    return config
+def create_bert(cache_dir=None):
+    """Creates a GPT2 model, config, and tokenizer from the given name and revision"""
+    from transformers import BertConfig
+
+    config = BertConfig.from_pretrained("./cs77_proj/bert_base/checkpoint-3848/config.json")
+    tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+    gpt = AutoModelForSequenceClassification.from_pretrained("./cs77_proj/bert_base/checkpoint-3848", config=config, cache_dir=cache_dir)
+    print("loaded model")
+    return config, tokenizer, gpt
+
+# params = return_params('best_model_json/distilbert.json', 0.001 )
+#params = return_params('best_model_json/distilbert.json', 1 )
+
+
+'''embeddings=None
+if(params['bert_tokens']):
+    train,val,test=createDatasetSplit(params)       #update
+else:
+    train,val,test,vocab_own=createDatasetSplit(params)
+    params['embed_size']=vocab_own.embeddings.shape[1]
+    params['vocab_size']=vocab_own.embeddings.shape[0]
+    embeddings=vocab_own.embeddings
+if(params['auto_weights']):
+    y_test = [ele[2] for ele in test] 
+#         print(y_test)
+    encoder = LabelEncoder()
+    encoder.classes_ = np.load(params['class_names'],allow_pickle=True)
+    params['weights']=class_weight.compute_class_weight('balanced',np.unique(y_test),y_test).astype('float32') 
+    #params['weights']=np.array([len(y_test)/y_test.count(encoder.classes_[0]),len(y_test)/y_test.count(encoder.classes_[1]),len(y_test)/y_test.count(encoder.classes_[2])]).astype('float32') 
+
+model=select_model(params,embeddings)
+model = model.eval()
+tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
+
+
+classes_ = np.load('Data/classes.npy')
+'''
+def main_function(text,label):
+    '''tokens = tokenizer.encode_plus(text)
+    input_ids = pad_sequences(torch.tensor(tokens['input_ids']).unsqueeze(0),maxlen=int(params['max_length']),\
+                               dtype="long", 
+                          value=0, truncating="post", padding="post")
+    # att_vals = pad_sequences(att_vals,maxlen=int(params['max_length']), dtype="float", 
+    #                       value=0.0, truncating="post", padding="post")
+    att_masks=custom_att_masks(input_ids)
+
+    outs = model(torch.tensor(input_ids), 
+            attention_mask=torch.tensor(att_masks, dtype=bool), 
+            labels=None,
+            device='cuda')
+    
+    text_tokens = tokenizer.convert_ids_to_tokens(input_ids.squeeze())
+
+    text_tokens_ = text_tokens[:len(tokens['input_ids'])]
+
+    print ('xyz')
+    print (outs[1][5].shape)
+    avg_attn = torch.mean(outs[1][5], dim=1)
+    avg_attn_np = avg_attn[0,0,:len(tokens['input_ids'])].detach().squeeze().numpy()
+    
+    logits = outs[0]
+    print (logits)
+    print (np.sum(avg_attn_np))
+    print (avg_attn_np)
+
+    pred = torch.argmax(logits)
+    pred_label = classes_[pred]
+    '''
+    ml_img_path,bo_img_path,atten_img_path = interpret(text,label)
+    ml_im = Image.open(ml_img_path)
+    bo_im = Image.open(bo_img_path)
+    atten_im = Image.open(atten_img_path)
+
+    yield ml_im, bo_im, atten_im
+
+    '''
+    sns.set_theme(rc={'figure.figsize':(30,1)})
+  
+    # creating subplot 
+    fig, ax = plt.subplots() 
+    
+    # drawing heatmap on current axes 
+    ax = sns.heatmap(np.expand_dims(avg_attn_np,0), annot= np.expand_dims(np.array(text_tokens_),0), \
+                                            fmt="", annot_kws={'size': 10}, cmap="magma") 
+
+    fig = ax.get_figure()
+    fig.savefig("out.png" ,bbox_inches='tight')
+
+    im = Image.open("out.png")
+
+    yield im
+
+    '''
+
+    #return list(zip(text_tokens_ , avg_attn_np)), pred_label
+    # return list(zip(text_tokens_[1:-1] , avg_attn_np[1:-1])) 
+     
+
+demo = gr.Interface(main_function,
+                    inputs="textbox",
+                    outputs="image",
+                    theme = 'compact')
+
+with gr.Blocks() as demo:
+    with gr.Tab("Text Input"):
+        text_input = gr.Textbox()
+        label_input = gr.Textbox()
+        text_button = gr.Button("Show")
+
+    with gr.Tab("Interpretability"):
+         with gr.Row():
+              image_output1 = gr.Image()
+              image_output2 = gr.Image()
+              image_output3 = gr.Image()
+
+    text_button.click(main_function, inputs=[text_input,label_input], outputs=[image_output1,image_output2,image_output3])
+
+
+
+
+if __name__ == "__main__":
+    demo.launch()

cs772_proj/bert_base/checkpoint-3848/config.json

@@ -0,0 +1,37 @@
+{
+  "_name_or_path": "bert-base-uncased",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "hate",
+    "1": "normal",
+    "2": "offense"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "hate": 0,
+    "normal": 1,
+    "offense": 2
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.39.3",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

cs772_proj/bert_base/checkpoint-3848/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f92169bcbaeee93e4c65d5f6b7af90505d8a754096d1b7d1ea70cf290cc79690
+size 437961724

cs772_proj/bert_base/checkpoint-3848/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:306504e0a7e6e3e27b15a81346a5f70e8941e6ec7085d33d70693b13cbba1e8b
+size 876044538

cs772_proj/bert_base/checkpoint-3848/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f8c99e888295714e206a5143fed689d23c7ae28a194ff83078714c2d99f94ab
+size 14244

cs772_proj/bert_base/checkpoint-3848/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9258b30f99447d96f005979b906a97fe44e711e5ecb53f5be292707492c5ef45
+size 1064

cs772_proj/bert_base/checkpoint-3848/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

cs772_proj/bert_base/checkpoint-3848/tokenizer_config.json

@@ -0,0 +1,55 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

cs772_proj/bert_base/checkpoint-3848/trainer_state.json

@@ -0,0 +1,85 @@
+{
+  "best_metric": 0.7247874736785889,
+  "best_model_checkpoint": "bert_base/checkpoint-1924",
+  "epoch": 4.0,
+  "eval_steps": 500,
+  "global_step": 3848,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.0,
+      "grad_norm": 13.14333724975586,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.8204,
+      "step": 962
+    },
+    {
+      "epoch": 1.0,
+      "eval_f1": 0.6669116037121897,
+      "eval_loss": 0.7362112402915955,
+      "eval_runtime": 2.6989,
+      "eval_samples_per_second": 712.147,
+      "eval_steps_per_second": 44.833,
+      "step": 962
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 15.062124252319336,
+      "learning_rate": 6e-06,
+      "loss": 0.664,
+      "step": 1924
+    },
+    {
+      "epoch": 2.0,
+      "eval_f1": 0.6714560068474462,
+      "eval_loss": 0.7247874736785889,
+      "eval_runtime": 1.9229,
+      "eval_samples_per_second": 999.531,
+      "eval_steps_per_second": 62.926,
+      "step": 1924
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 30.878219604492188,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.5662,
+      "step": 2886
+    },
+    {
+      "epoch": 3.0,
+      "eval_f1": 0.6630607481681304,
+      "eval_loss": 0.7806704044342041,
+      "eval_runtime": 2.4376,
+      "eval_samples_per_second": 788.47,
+      "eval_steps_per_second": 49.638,
+      "step": 2886
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 21.18539810180664,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.4919,
+      "step": 3848
+    },
+    {
+      "epoch": 4.0,
+      "eval_f1": 0.6878731135692044,
+      "eval_loss": 0.7753087282180786,
+      "eval_runtime": 2.3502,
+      "eval_samples_per_second": 817.816,
+      "eval_steps_per_second": 51.486,
+      "step": 3848
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 4810,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 500,
+  "total_flos": 1892113337449692.0,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

cs772_proj/bert_base/checkpoint-3848/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f562e068a01b97d232a8b2fbb9f51b80ec2d1eedd0dd5a99be9c3f9af0bcbeb1
+size 4856

cs772_proj/cs772_bert/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

cs772_proj/cs772_bert/README.md

@@ -0,0 +1,13 @@
+---
+title: Cs772 Bert
+emoji: 🌖
+colorFrom: indigo
+colorTo: gray
+sdk: gradio
+sdk_version: 4.29.0
+app_file: app.py
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

cs772_proj/requirements.txt

@@ -0,0 +1,123 @@
+accelerate==0.29.2
+aiohttp==3.9.4
+aiosignal==1.3.1
+asttokens 
+async-timeout==4.0.3
+attrs==23.2.0
+backcall==0.2.0
+beautifulsoup4==4.12.3
+bleach==6.1.0
+certifi==2024.2.2
+charset-normalizer==3.3.2
+comm 
+contourpy==1.2.1
+cycler==0.12.1
+datasets==2.18.0
+debugpy
+decorator 
+defusedxml==0.7.1
+dill==0.3.8
+docopt==0.6.2
+exceptiongroup 
+executing 
+fastjsonschema==2.19.1
+filelock==3.13.4
+fonttools==4.51.0
+frozenlist==1.4.1
+fsspec==2023.6.0
+huggingface-hub==0.20.3
+idna==3.7
+importlib_metadata 
+ipykernel 
+ipython==8.12.3
+ipywidgets==8.1.2
+jedi
+Jinja2==3.1.3
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+jupyter_client
+jupyter_core 
+jupyterlab_pygments==0.3.0
+jupyterlab_widgets==3.0.10
+kiwisolver==1.4.5
+MarkupSafe==2.1.5
+matplotlib==3.8.4
+matplotlib-inline 
+mistune==3.0.2
+mizani==0.11.1
+mpmath==1.3.0
+multidict==6.0.5
+multiprocess==0.70.16
+nbclient==0.10.0
+nbconvert==7.16.3
+nbformat==5.10.4
+nest_asyncio 
+networkx==3.3
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.19.3
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nvtx-cu12==12.1.105
+packaging 
+pandas==2.2.2
+pandocfilters==1.5.1
+parso 
+patsy==0.5.6
+pexpect 
+pickleshare 
+pillow==10.3.0
+pipreqs==0.5.0
+platformdirs
+plotnine==0.13.4
+prompt-toolkit 
+protobuf==5.26.1
+psutil 
+ptyprocess 
+pure-eval 
+pyarrow==15.0.2
+pyarrow-hotfix==0.6
+Pygments 
+pyparsing==3.1.2
+python-dateutil==2.8.2
+pytz==2023.4
+pyvene==0.1.1
+PyYAML==6.0.1
+pyzmq 
+referencing==0.34.0
+regex==2023.12.25
+requests==2.31.0
+rpds-py==0.18.0
+safetensors==0.4.3
+scipy==1.13.0
+sentencepiece==0.2.0
+six 
+soupsieve==2.5
+stack-data 
+statsmodels==0.14.1
+sympy==1.12
+tinycss2==1.2.1
+tokenizers==0.15.2
+torch==2.2.2
+tornado 
+tqdm==4.66.2
+traitlets 
+transformers==4.39.3
+triton==2.2.0
+typing_extensions 
+tzdata==2024.1
+urllib3==2.0.7
+wcwidth 
+webencodings==0.5.1
+widgetsnbextension==4.0.10
+xxhash==3.4.1
+yarg==0.1.9
+yarl==1.9.4
+zipp 

cs772_proj/tutorial_data/pyvene_rome_attention_output.csv

@@ -0,0 +1,73 @@
+,layer,pos,prob
+0,0,0,0.008943566121160984
+1,0,1,0.010685674846172333
+2,0,2,0.015678975731134415
+3,0,3,0.01495782658457756
+4,0,4,0.01689751259982586
+5,0,5,0.012341184541583061
+6,1,0,0.00910158734768629
+7,1,1,0.011121801100671291
+8,1,2,0.015446535311639309
+9,1,3,0.014828759245574474
+10,1,4,0.01610460691154003
+11,1,5,0.012241763062775135
+12,2,0,0.009373819455504417
+13,2,1,0.011316204443573952
+14,2,2,0.01544259861111641
+15,2,3,0.014399203471839428
+16,2,4,0.015949850901961327
+17,2,5,0.012191198766231537
+18,3,0,0.008611239492893219
+19,3,1,0.01138687040656805
+20,3,2,0.015247474424540997
+21,3,3,0.013744203373789787
+22,3,4,0.014804143458604813
+23,3,5,0.011855616234242916
+24,4,0,0.009979105554521084
+25,4,1,0.011923858895897865
+26,4,2,0.015469703823328018
+27,4,3,0.012778976932168007
+28,4,4,0.015446675941348076
+29,4,5,0.01213959138840437
+30,5,0,0.010452548041939735
+31,5,1,0.011575913988053799
+32,5,2,0.014227043837308884
+33,5,3,0.013159635476768017
+34,5,4,0.016256239265203476
+35,5,5,0.01196625828742981
+36,6,0,0.009859082289040089
+37,6,1,0.011729804798960686
+38,6,2,0.013667005114257336
+39,6,3,0.012512612156569958
+40,6,4,0.015985535457730293
+41,6,5,0.011508451774716377
+42,7,0,0.00967455469071865
+43,7,1,0.012198343873023987
+44,7,2,0.013812437653541565
+45,7,3,0.012038654647767544
+46,7,4,0.014745757915079594
+47,7,5,0.011055140756070614
+48,8,0,0.01034906692802906
+49,8,1,0.011351429857313633
+50,8,2,0.013925875537097454
+51,8,3,0.012646789662539959
+52,8,4,0.01411098800599575
+53,8,5,0.011073073372244835
+54,9,0,0.013398675248026848
+55,9,1,0.011368145234882832
+56,9,2,0.013541489839553833
+57,9,3,0.013448523357510567
+58,9,4,0.013419842347502708
+59,9,5,0.011098676361143589
+60,10,0,0.013398675248026848
+61,10,1,0.012150835245847702
+62,10,2,0.014172807335853577
+63,10,3,0.012981802225112915
+64,10,4,0.013179052621126175
+65,10,5,0.01129151601344347
+66,11,0,0.013398675248026848
+67,11,1,0.01180819422006607
+68,11,2,0.013985361903905869
+69,11,3,0.012903643772006035
+70,11,4,0.012925814837217331
+71,11,5,0.011390508152544498

cs772_proj/tutorial_data/pyvene_rome_block_output.csv

@@ -0,0 +1,73 @@
+,layer,pos,prob
+0,0,0,0.009189224801957607
+1,0,1,0.011389641091227531
+2,0,2,0.0162599328905344
+3,0,3,0.015484759584069252
+4,0,4,0.015411637723445892
+5,0,5,0.012490469962358475
+6,1,0,0.00770866172388196
+7,1,1,0.011720607057213783
+8,1,2,0.015047593042254448
+9,1,3,0.014841136522591114
+10,1,4,0.017443198710680008
+11,1,5,0.011815374717116356
+12,2,0,0.008566385135054588
+13,2,1,0.01111418567597866
+14,2,2,0.01541436929255724
+15,2,3,0.014069304801523685
+16,2,4,0.016460780054330826
+17,2,5,0.0121275270357728
+18,3,0,0.009172435849905014
+19,3,1,0.011352349072694778
+20,3,2,0.013832006603479385
+21,3,3,0.014499133452773094
+22,3,4,0.01608533412218094
+23,3,5,0.011975396424531937
+24,4,0,0.009531590156257153
+25,4,1,0.011509168893098831
+26,4,2,0.012929881922900677
+27,4,3,0.013458534143865108
+28,4,4,0.015189730562269688
+29,4,5,0.011921005323529243
+30,5,0,0.009805092588067055
+31,5,1,0.011592468246817589
+32,5,2,0.013322774320840836
+33,5,3,0.01245818566530943
+34,5,4,0.013958347029983997
+35,5,5,0.012003983370959759
+36,6,0,0.01007422897964716
+37,6,1,0.010900546796619892
+38,6,2,0.01368661504238844
+39,6,3,0.01260523870587349
+40,6,4,0.013009610585868359
+41,6,5,0.012099610641598701
+42,7,0,0.010249304585158825
+43,7,1,0.010945979505777359
+44,7,2,0.013585647568106651
+45,7,3,0.013284442014992237
+46,7,4,0.012696263380348682
+47,7,5,0.012064820155501366
+48,8,0,0.009416966699063778
+49,8,1,0.011989694088697433
+50,8,2,0.01403607614338398
+51,8,3,0.012878036126494408
+52,8,4,0.012870670296251774
+53,8,5,0.011852720752358437
+54,9,0,0.009302603080868721
+55,9,1,0.011646227911114693
+56,9,2,0.013754121959209442
+57,9,3,0.01287330687046051
+58,9,4,0.012776567600667477
+59,9,5,0.011404040269553661
+60,10,0,0.009880894795060158
+61,10,1,0.011837868951261044
+62,10,2,0.013910908252000809
+63,10,3,0.012473315000534058
+64,10,4,0.012750478461384773
+65,10,5,0.011884817853569984
+66,11,0,0.013398675248026848
+67,11,1,0.012010819278657436
+68,11,2,0.012010819278657436
+69,11,3,0.012010819278657436
+70,11,4,0.012010819278657436
+71,11,5,0.012010819278657436

cs772_proj/tutorial_data/pyvene_rome_mlp_activation.csv

@@ -0,0 +1,73 @@
+,layer,pos,prob
+0,0,0,0.0075546312145888805
+1,0,1,0.011380046606063843
+2,0,2,0.01438708696514368
+3,0,3,0.015439963899552822
+4,0,4,0.015718040987849236
+5,0,5,0.012858170084655285
+6,1,0,0.0077091907151043415
+7,1,1,0.011459099128842354
+8,1,2,0.014624425210058689
+9,1,3,0.015179034322500229
+10,1,4,0.015754742547869682
+11,1,5,0.012920349836349487
+12,2,0,0.007979463785886765
+13,2,1,0.011575750075280666
+14,2,2,0.014750510454177856
+15,2,3,0.014939533546566963
+16,2,4,0.01672947406768799
+17,2,5,0.012872524559497833
+18,3,0,0.008789473213255405
+19,3,1,0.011063076555728912
+20,3,2,0.01672506332397461
+21,3,3,0.012915139086544514
+22,3,4,0.01752210408449173
+23,3,5,0.012578015215694904
+24,4,0,0.009665396064519882
+25,4,1,0.011315570212900639
+26,4,2,0.016729004681110382
+27,4,3,0.012932662852108479
+28,4,4,0.017836520448327065
+29,4,5,0.012803135439753532
+30,5,0,0.010207359679043293
+31,5,1,0.01099418569356203
+32,5,2,0.01522758323699236
+33,5,3,0.012608421966433525
+34,5,4,0.01690223254263401
+35,5,5,0.01230985764414072
+36,6,0,0.009948461316525936
+37,6,1,0.011443679220974445
+38,6,2,0.013499817810952663
+39,6,3,0.012555226683616638
+40,6,4,0.01549310702830553
+41,6,5,0.011905322782695293
+42,7,0,0.009184295311570168
+43,7,1,0.011352204717695713
+44,7,2,0.01403868943452835
+45,7,3,0.012666325084865093
+46,7,4,0.013838390819728374
+47,7,5,0.011248479597270489
+48,8,0,0.010832141153514385
+49,8,1,0.011385922320187092
+50,8,2,0.01583883911371231
+51,8,3,0.01382371224462986
+52,8,4,0.014275728724896908
+53,8,5,0.011227857321500778
+54,9,0,0.013241364620625973
+55,9,1,0.01146922167390585
+56,9,2,0.015066420659422874
+57,9,3,0.013642949052155018
+58,9,4,0.013898820616304874
+59,9,5,0.011261279694736004
+60,10,0,0.013216082938015461
+61,10,1,0.012054135091602802
+62,10,2,0.014480901882052422
+63,10,3,0.012983473017811775
+64,10,4,0.012887177988886833
+65,10,5,0.011302494443953037
+66,11,0,0.013019828125834465
+67,11,1,0.01216293778270483
+68,11,2,0.01321493461728096
+69,11,3,0.012598911300301552
+70,11,4,0.013332013040781021
+71,11,5,0.011366385966539383