{ "name": "InterpBench", "version": "1.0.0", "description": "A benchmark of transformers with known circuits for evaluating mechanistic interpretability techniques.", "license": "https://creativecommons.org/licenses/by/4.0/", "url": "https://huggingface.co/cybershiptrooper/InterpBench", "cases": [ { "case_id": "11", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/11", "task_description": "Counts the number of words in a sequence based on their length.", "vocab": [ "J", "LB", "TPSI", "V", "b", "no", "oCLrZaW", "poiVg" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/edges.pkl" }, { "file_name": "ll_model_510.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model_510.pth" }, { "file_name": "ll_model_cfg_510.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model_cfg_510.pkl" }, { "file_name": "meta_510.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/meta_510.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 12, "n_ctx": 10, "d_head": 3, "model_name": "custom", "n_heads": 4, "d_mlp": 48, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1460593486680443, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 5, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 3456, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 500, "act_fn": "gelu", "clip_grad_norm": 1.0, "lr_scheduler": "" } }, { "case_id": "13", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/13", "task_description": "Analyzes the trend (increasing, decreasing, constant) of numeric tokens.", "vocab": [ 0, 1, 2 ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/edges.pkl" }, { "file_name": "ll_model_510.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model_510.pth" }, { "file_name": "ll_model_cfg_510.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model_cfg_510.pkl" }, { "file_name": "meta_510.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/meta_510.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 20, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 80, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "bidirectional", "attn_only": false, "seed": 0, "initializer_range": 0.1460593486680443, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 9600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 500, "act_fn": "gelu", "clip_grad_norm": 1.0, "lr_scheduler": "" } }, { "case_id": "18", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/18", "task_description": "Classify each token based on its frequency as 'rare', 'common', or 'frequent'.", "vocab": [ "a", "b", "c", "d", "e" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/edges.pkl" }, { "file_name": "ll_model_510.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model_510.pth" }, { "file_name": "ll_model_cfg_510.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model_cfg_510.pkl" }, { "file_name": "meta_510.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/meta_510.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 12, "n_ctx": 10, "d_head": 3, "model_name": "custom", "n_heads": 4, "d_mlp": 48, "act_fn": "gelu", "d_vocab": 7, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "bidirectional", "attn_only": false, "seed": 0, "initializer_range": 0.12344267996967354, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 3456, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "19", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/19", "task_description": "Removes consecutive duplicate tokens from a sequence.", "vocab": [ "a", "b", "c" ], "max_seq_len": 15, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/edges.pkl" }, { "file_name": "ll_model_510.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model_510.pth" }, { "file_name": "ll_model_cfg_510.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model_cfg_510.pkl" }, { "file_name": "meta_510.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/meta_510.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 32, "n_ctx": 15, "d_head": 8, "model_name": "custom", "n_heads": 4, "d_mlp": 128, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.15689290811054724, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 24576, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "20", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/20", "task_description": "Detect spam messages based on appearance of spam keywords.", "vocab": [ "J", "LB", "TPSI", "V", "b", "click", "no", "now", "oCLrZaW", "offer", "poiVg", "spam" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/edges.pkl" }, { "file_name": "ll_model_1110.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model_1110.pth" }, { "file_name": "ll_model_cfg_1110.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model_cfg_1110.pkl" }, { "file_name": "meta_1110.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/meta_1110.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 4, "n_ctx": 10, "d_head": 1, "model_name": "custom", "n_heads": 4, "d_mlp": 16, "act_fn": "gelu", "d_vocab": 14, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cuda", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.16, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 2, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 384, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 1.0, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "21", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/21", "task_description": "Extract unique tokens from a string", "vocab": [ "a", "b", "c" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/edges.pkl" }, { "file_name": "ll_model_510.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model_510.pth" }, { "file_name": "ll_model_cfg_510.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model_cfg_510.pkl" }, { "file_name": "meta_510.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/meta_510.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 20, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 80, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1885618083164127, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 9600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 500, "act_fn": "gelu", "clip_grad_norm": 1.0, "lr_scheduler": "" } }, { "case_id": "24", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/24", "task_description": "Identifies the first occurrence of each token in a sequence.", "vocab": [ "a", "b", "c" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/24/edges.pkl" }, { "file_name": "ll_model_510.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/24/ll_model_510.pth" }, { "file_name": "ll_model_cfg_510.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/24/ll_model_cfg_510.pkl" }, { "file_name": "meta_510.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/24/meta_510.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 20, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 80, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1885618083164127, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 9600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 500, "act_fn": "gelu", "clip_grad_norm": 1.0, "lr_scheduler": "" } }, { "case_id": "3", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/3", "task_description": "Returns the fraction of 'x' in the input up to the i-th position for all i.", "vocab": [ "a", "b", "c", "x" ], "max_seq_len": 5, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/edges.pkl" }, { "file_name": "ll_model_10110.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model_10110.pth" }, { "file_name": "ll_model_cfg_10110.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model_cfg_10110.pkl" }, { "file_name": "meta_10110.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/meta_10110.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 12, "n_ctx": 5, "d_head": 3, "model_name": "custom", "n_heads": 4, "d_mlp": 48, "act_fn": "gelu", "d_vocab": 6, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.22188007849009167, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 1, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 3456, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 10.0, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "33", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/33", "task_description": "Checks if each token's length is odd or even.", "vocab": [ "J", "LB", "TPSI", "V", "b", "no", "oCLrZaW", "poiVg" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/edges.pkl" }, { "file_name": "ll_model_510.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model_510.pth" }, { "file_name": "ll_model_cfg_510.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model_cfg_510.pkl" }, { "file_name": "meta_510.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/meta_510.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 4, "n_ctx": 10, "d_head": 1, "model_name": "custom", "n_heads": 4, "d_mlp": 16, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.17457431218879393, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 2, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 384, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "34", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/34", "task_description": "Calculate the ratio of vowels to consonants in each word.", "vocab": [ "J", "LB", "TPSI", "V", "b", "no", "oCLrZaW", "poiVg" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/edges.pkl" }, { "file_name": "ll_model_510.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model_510.pth" }, { "file_name": "ll_model_cfg_510.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model_cfg_510.pkl" }, { "file_name": "meta_510.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/meta_510.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 4, "n_ctx": 10, "d_head": 1, "model_name": "custom", "n_heads": 4, "d_mlp": 16, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.16329931618554522, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 5, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 384, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "35", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/35", "task_description": "Alternates capitalization of each character in words.", "vocab": [ "J", "LB", "TPSI", "V", "b", "no", "oCLrZaW", "poiVg" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/edges.pkl" }, { "file_name": "ll_model_510.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model_510.pth" }, { "file_name": "ll_model_cfg_510.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model_cfg_510.pkl" }, { "file_name": "meta_510.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/meta_510.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 4, "n_ctx": 10, "d_head": 1, "model_name": "custom", "n_heads": 4, "d_mlp": 16, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1539600717839002, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 8, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 384, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "36", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/36", "task_description": "Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.", "vocab": [ "\ud83d\udcd8", "\ud83d\ude0a", "\ud83d\ude22" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/edges.pkl" }, { "file_name": "ll_model_10110.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model_10110.pth" }, { "file_name": "ll_model_cfg_10110.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model_cfg_10110.pkl" }, { "file_name": "meta_10110.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/meta_10110.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 4, "n_ctx": 10, "d_head": 1, "model_name": "custom", "n_heads": 4, "d_mlp": 16, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cuda", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.19402850002906638, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 3, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 384, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 10.0, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "37", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/37", "task_description": "Reverses each word in the sequence except for specified exclusions.", "vocab": [ "J", "LB", "TPSI", "V", "b", "no", "oCLrZaW", "poiVg" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/edges.pkl" }, { "file_name": "ll_model_510.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model_510.pth" }, { "file_name": "ll_model_cfg_510.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model_cfg_510.pkl" }, { "file_name": "meta_510.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/meta_510.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 4, "n_ctx": 10, "d_head": 1, "model_name": "custom", "n_heads": 4, "d_mlp": 16, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1539600717839002, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 8, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 384, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "38", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/38", "task_description": "Checks if tokens alternate between two types.", "vocab": [ "a", "b", "c" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/38/edges.pkl" }, { "file_name": "ll_model_510.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/38/ll_model_510.pth" }, { "file_name": "ll_model_cfg_510.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/38/ll_model_cfg_510.pkl" }, { "file_name": "meta_510.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/38/meta_510.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 20, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 80, "act_fn": "gelu", "d_vocab": 5, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.1539600717839002, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 2, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 9600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "4", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/4", "task_description": "Return fraction of previous open tokens minus the fraction of close tokens.", "vocab": [ "(", ")", "a", "b", "c" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/edges.pkl" }, { "file_name": "ll_model_510.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model_510.pth" }, { "file_name": "ll_model_cfg_510.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model_cfg_510.pkl" }, { "file_name": "meta_510.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/meta_510.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 20, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 80, "act_fn": "gelu", "d_vocab": 7, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.17056057308448835, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 1, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 9600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.001, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 2000, "act_fn": "gelu", "clip_grad_norm": 0.1, "lr_scheduler": "" } }, { "case_id": "8", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/8", "task_description": "Fills gaps between tokens with a specified filler.", "vocab": [ "J", "LB", "TPSI", "V", "b", "no", "oCLrZaW", "poiVg" ], "max_seq_len": 10, "min_seq_len": 4, "files": [ { "file_name": "edges.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/8/edges.pkl" }, { "file_name": "ll_model_510.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/8/ll_model_510.pth" }, { "file_name": "ll_model_cfg_510.pkl", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/8/ll_model_cfg_510.pkl" }, { "file_name": "meta_510.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/8/meta_510.json" } ], "transformer_cfg": { "n_layers": 2, "d_model": 20, "n_ctx": 10, "d_head": 5, "model_name": "custom", "n_heads": 4, "d_mlp": 80, "act_fn": "gelu", "d_vocab": 10, "eps": 1e-05, "use_attn_result": true, "use_attn_scale": true, "use_split_qkv_input": true, "use_hook_mlp_in": true, "use_attn_in": false, "use_local_attn": false, "original_architecture": null, "from_checkpoint": false, "checkpoint_index": null, "checkpoint_label_type": null, "checkpoint_value": null, "tokenizer_name": null, "window_size": null, "attn_types": null, "init_mode": "gpt2", "normalization_type": null, "device": "cpu", "n_devices": 1, "attention_dir": "causal", "attn_only": false, "seed": 0, "initializer_range": 0.13333333333333333, "init_weights": true, "scale_attn_by_inverse_layer_idx": false, "positional_embedding_type": "standard", "final_rms": false, "d_vocab_out": 8, "parallel_attn_mlp": false, "rotary_dim": null, "n_params": 9600, "use_hook_tokens": false, "gated_mlp": false, "default_prepend_bos": true, "dtype": "torch.float32", "tokenizer_prepends_bos": null, "n_key_value_heads": null, "post_embedding_ln": false, "rotary_base": 10000, "trust_remote_code": false, "rotary_adjacent_pairs": false }, "training_args": { "atol": 0.05, "lr": 0.01, "use_single_loss": false, "iit_weight": 1.0, "behavior_weight": 1.0, "strict_weight": 0.4, "epochs": 500, "act_fn": "gelu", "clip_grad_norm": 1.0, "lr_scheduler": "" } }, { "case_id": "ioi", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi", "task_description": "Indirect object identification", "max_seq_len": 16, "min_seq_len": 16, "files": [ { "file_name": "corr_100_100_40.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/corr_100_100_40.json" }, { "file_name": "ll_model_100_100_40.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model_100_100_40.pth" } ] }, { "case_id": "ioi_next_token", "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi_next_token", "task_description": "Indirect object identification", "max_seq_len": 16, "min_seq_len": 16, "files": [ { "file_name": "corr_100_100_40.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/corr_100_100_40.json" }, { "file_name": "ll_model_100_100_40.pth", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model_100_100_40.pth" }, { "file_name": "training_args.json", "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/training_args.json" } ] } ] }