Add metadata files (#1)

- Add metadata files (a1d74d9edcc5fe196be04281417a40c81dbfae63)


Co-authored-by: Ivan Arcuschin <[email protected]>

benchmark_cases_metadata.csv

@@ -0,0 +1,19 @@
+case_id,task_description,max_seq_len,min_seq_len,training_args.atol,training_args.lr,training_args.use_single_loss,training_args.iit_weight,training_args.behavior_weight,training_args.strict_weight,training_args.epochs,training_args.act_fn,training_args.clip_grad_norm,training_args.lr_scheduler,transformer_cfg.n_layers,transformer_cfg.d_model,transformer_cfg.n_ctx,transformer_cfg.d_head,transformer_cfg.model_name,transformer_cfg.n_heads,transformer_cfg.d_mlp,transformer_cfg.act_fn,transformer_cfg.d_vocab,transformer_cfg.eps,transformer_cfg.use_attn_result,transformer_cfg.use_attn_scale,transformer_cfg.use_split_qkv_input,transformer_cfg.use_hook_mlp_in,transformer_cfg.use_attn_in,transformer_cfg.use_local_attn,transformer_cfg.original_architecture,transformer_cfg.from_checkpoint,transformer_cfg.checkpoint_index,transformer_cfg.checkpoint_label_type,transformer_cfg.checkpoint_value,transformer_cfg.tokenizer_name,transformer_cfg.window_size,transformer_cfg.attn_types,transformer_cfg.init_mode,transformer_cfg.normalization_type,transformer_cfg.device,transformer_cfg.n_devices,transformer_cfg.attention_dir,transformer_cfg.attn_only,transformer_cfg.seed,transformer_cfg.initializer_range,transformer_cfg.init_weights,transformer_cfg.scale_attn_by_inverse_layer_idx,transformer_cfg.positional_embedding_type,transformer_cfg.final_rms,transformer_cfg.d_vocab_out,transformer_cfg.parallel_attn_mlp,transformer_cfg.rotary_dim,transformer_cfg.n_params,transformer_cfg.use_hook_tokens,transformer_cfg.gated_mlp,transformer_cfg.default_prepend_bos,transformer_cfg.dtype,transformer_cfg.tokenizer_prepends_bos,transformer_cfg.n_key_value_heads,transformer_cfg.post_embedding_ln,transformer_cfg.rotary_base,transformer_cfg.trust_remote_code,transformer_cfg.rotary_adjacent_pairs
+11,Counts the number of words in a sequence based on their length.,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,12.0,10.0,3.0,custom,4.0,48.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1460593486680443,True,False,standard,False,5.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
+13,"Analyzes the trend (increasing, decreasing, constant) of numeric tokens.",10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,bidirectional,False,0.0,0.1460593486680443,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+18,"Classify each token based on its frequency as 'rare', 'common', or 'frequent'.",10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,12.0,10.0,3.0,custom,4.0,48.0,gelu,7.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,bidirectional,False,0.0,0.12344267996967354,True,False,standard,False,3.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
+19,Removes consecutive duplicate tokens from a sequence.,15.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,32.0,15.0,8.0,custom,4.0,128.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.15689290811054724,True,False,standard,False,3.0,False,,24576.0,False,False,True,torch.float32,,,False,10000.0,False,False
+20,Detect spam messages based on appearance of spam keywords.,10.0,4.0,0.05,0.001,False,1.0,1.0,1.0,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,14.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cuda,1.0,causal,False,0.0,0.16,True,False,standard,False,2.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+21,Extract unique tokens from a string,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1885618083164127,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+24,Identifies the first occurrence of each token in a sequence.,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1885618083164127,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+3,Returns the fraction of 'x' in the input up to the i-th position for all i.,5.0,4.0,0.05,0.001,False,1.0,1.0,10.0,2000.0,gelu,0.1,,2.0,12.0,5.0,3.0,custom,4.0,48.0,gelu,6.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.22188007849009167,True,False,standard,False,1.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
+33,Checks if each token's length is odd or even.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.17457431218879393,True,False,standard,False,2.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+34,Calculate the ratio of vowels to consonants in each word.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.16329931618554522,True,False,standard,False,5.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+35,Alternates capitalization of each character in words.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,8.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+36,"Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.",10.0,4.0,0.05,0.001,False,1.0,1.0,10.0,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cuda,1.0,causal,False,0.0,0.19402850002906638,True,False,standard,False,3.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+37,Reverses each word in the sequence except for specified exclusions.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,8.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+38,Checks if tokens alternate between two types.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,2.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+4,Return fraction of previous open tokens minus the fraction of close tokens.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,7.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.17056057308448835,True,False,standard,False,1.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+8,Fills gaps between tokens with a specified filler.,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.13333333333333333,True,False,standard,False,8.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+ioi,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+ioi_next_token,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,

benchmark_cases_metadata.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01f473326ab84033784efd049808e3f0395bf577e23311a60f9b05eff32f5ca3
+size 49942

benchmark_metadata.json

@@ -0,0 +1,1362 @@
+{
+ "name": "InterpBench",
+ "version": "1.0.0",
+ "description": "A benchmark of transformers with known circuits for evaluating mechanistic interpretability techniques.",
+ "cases": [
+ {
+ "case_id": "11",
+ "files": [
+ "edges.pkl",
+ "ll_model_510.pth",
+ "ll_model_cfg_510.pkl",
+ "meta_510.json"
+ ],
+ "task_description": "Counts the number of words in a sequence based on their length.",
+ "vocab": [
+ "J",
+ "oCLrZaW",
+ "no",
+ "poiVg",
+ "V",
+ "b",
+ "LB",
+ "TPSI"
+ ],
+ "max_seq_len": 10,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 12,
+ "n_ctx": 10,
+ "d_head": 3,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 48,
+ "act_fn": "gelu",
+ "d_vocab": 10,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cpu",
+ "n_devices": 1,
+ "attention_dir": "causal",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.1460593486680443,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 5,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 3456,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.01,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 0.4,
+ "epochs": 500,
+ "act_fn": "gelu",
+ "clip_grad_norm": 1.0,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "13",
+ "files": [
+ "edges.pkl",
+ "ll_model_510.pth",
+ "ll_model_cfg_510.pkl",
+ "meta_510.json"
+ ],
+ "task_description": "Analyzes the trend (increasing, decreasing, constant) of numeric tokens.",
+ "vocab": [
+ 0,
+ 1,
+ 2
+ ],
+ "max_seq_len": 10,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 20,
+ "n_ctx": 10,
+ "d_head": 5,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 80,
+ "act_fn": "gelu",
+ "d_vocab": 5,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cpu",
+ "n_devices": 1,
+ "attention_dir": "bidirectional",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.1460593486680443,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 3,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 9600,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.01,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 0.4,
+ "epochs": 500,
+ "act_fn": "gelu",
+ "clip_grad_norm": 1.0,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "18",
+ "files": [
+ "edges.pkl",
+ "ll_model_510.pth",
+ "ll_model_cfg_510.pkl",
+ "meta_510.json"
+ ],
+ "task_description": "Classify each token based on its frequency as 'rare', 'common', or 'frequent'.",
+ "vocab": [
+ "c",
+ "e",
+ "b",
+ "d",
+ "a"
+ ],
+ "max_seq_len": 10,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 12,
+ "n_ctx": 10,
+ "d_head": 3,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 48,
+ "act_fn": "gelu",
+ "d_vocab": 7,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cpu",
+ "n_devices": 1,
+ "attention_dir": "bidirectional",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.12344267996967354,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 3,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 3456,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.001,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 0.4,
+ "epochs": 2000,
+ "act_fn": "gelu",
+ "clip_grad_norm": 0.1,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "19",
+ "files": [
+ "edges.pkl",
+ "ll_model_510.pth",
+ "ll_model_cfg_510.pkl",
+ "meta_510.json"
+ ],
+ "task_description": "Removes consecutive duplicate tokens from a sequence.",
+ "vocab": [
+ "b",
+ "a",
+ "c"
+ ],
+ "max_seq_len": 15,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 32,
+ "n_ctx": 15,
+ "d_head": 8,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 128,
+ "act_fn": "gelu",
+ "d_vocab": 5,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cpu",
+ "n_devices": 1,
+ "attention_dir": "causal",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.15689290811054724,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 3,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 24576,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.001,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 0.4,
+ "epochs": 2000,
+ "act_fn": "gelu",
+ "clip_grad_norm": 0.1,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "20",
+ "files": [
+ "edges.pkl",
+ "ll_model_1110.pth",
+ "ll_model_cfg_1110.pkl",
+ "meta_1110.json"
+ ],
+ "task_description": "Detect spam messages based on appearance of spam keywords.",
+ "vocab": [
+ "J",
+ "spam",
+ "offer",
+ "click",
+ "oCLrZaW",
+ "no",
+ "poiVg",
+ "V",
+ "b",
+ "LB",
+ "now",
+ "TPSI"
+ ],
+ "max_seq_len": 10,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 4,
+ "n_ctx": 10,
+ "d_head": 1,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 16,
+ "act_fn": "gelu",
+ "d_vocab": 14,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cuda",
+ "n_devices": 1,
+ "attention_dir": "causal",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.16,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 2,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 384,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.001,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 1.0,
+ "epochs": 2000,
+ "act_fn": "gelu",
+ "clip_grad_norm": 0.1,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "21",
+ "files": [
+ "edges.pkl",
+ "ll_model_510.pth",
+ "ll_model_cfg_510.pkl",
+ "meta_510.json"
+ ],
+ "task_description": "Extract unique tokens from a string",
+ "vocab": [
+ "b",
+ "a",
+ "c"
+ ],
+ "max_seq_len": 10,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 20,
+ "n_ctx": 10,
+ "d_head": 5,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 80,
+ "act_fn": "gelu",
+ "d_vocab": 5,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cpu",
+ "n_devices": 1,
+ "attention_dir": "causal",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.1885618083164127,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 3,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 9600,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.01,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 0.4,
+ "epochs": 500,
+ "act_fn": "gelu",
+ "clip_grad_norm": 1.0,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "24",
+ "files": [
+ "edges.pkl",
+ "ll_model_510.pth",
+ "ll_model_cfg_510.pkl",
+ "meta_510.json"
+ ],
+ "task_description": "Identifies the first occurrence of each token in a sequence.",
+ "vocab": [
+ "b",
+ "a",
+ "c"
+ ],
+ "max_seq_len": 10,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 20,
+ "n_ctx": 10,
+ "d_head": 5,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 80,
+ "act_fn": "gelu",
+ "d_vocab": 5,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cpu",
+ "n_devices": 1,
+ "attention_dir": "causal",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.1885618083164127,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 3,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 9600,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.01,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 0.4,
+ "epochs": 500,
+ "act_fn": "gelu",
+ "clip_grad_norm": 1.0,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "3",
+ "files": [
+ "edges.pkl",
+ "ll_model_10110.pth",
+ "ll_model_cfg_10110.pkl",
+ "meta_10110.json"
+ ],
+ "task_description": "Returns the fraction of 'x' in the input up to the i-th position for all i.",
+ "vocab": [
+ "x",
+ "b",
+ "a",
+ "c"
+ ],
+ "max_seq_len": 5,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 12,
+ "n_ctx": 5,
+ "d_head": 3,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 48,
+ "act_fn": "gelu",
+ "d_vocab": 6,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cpu",
+ "n_devices": 1,
+ "attention_dir": "causal",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.22188007849009167,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 1,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 3456,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.001,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 10.0,
+ "epochs": 2000,
+ "act_fn": "gelu",
+ "clip_grad_norm": 0.1,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "33",
+ "files": [
+ "edges.pkl",
+ "ll_model_510.pth",
+ "ll_model_cfg_510.pkl",
+ "meta_510.json"
+ ],
+ "task_description": "Checks if each token's length is odd or even.",
+ "vocab": [
+ "J",
+ "oCLrZaW",
+ "no",
+ "poiVg",
+ "V",
+ "b",
+ "LB",
+ "TPSI"
+ ],
+ "max_seq_len": 10,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 4,
+ "n_ctx": 10,
+ "d_head": 1,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 16,
+ "act_fn": "gelu",
+ "d_vocab": 10,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cpu",
+ "n_devices": 1,
+ "attention_dir": "causal",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.17457431218879393,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 2,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 384,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.001,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 0.4,
+ "epochs": 2000,
+ "act_fn": "gelu",
+ "clip_grad_norm": 0.1,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "34",
+ "files": [
+ "edges.pkl",
+ "ll_model_510.pth",
+ "ll_model_cfg_510.pkl",
+ "meta_510.json"
+ ],
+ "task_description": "Calculate the ratio of vowels to consonants in each word.",
+ "vocab": [
+ "J",
+ "oCLrZaW",
+ "no",
+ "poiVg",
+ "V",
+ "b",
+ "LB",
+ "TPSI"
+ ],
+ "max_seq_len": 10,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 4,
+ "n_ctx": 10,
+ "d_head": 1,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 16,
+ "act_fn": "gelu",
+ "d_vocab": 10,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cpu",
+ "n_devices": 1,
+ "attention_dir": "causal",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.16329931618554522,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 5,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 384,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.001,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 0.4,
+ "epochs": 2000,
+ "act_fn": "gelu",
+ "clip_grad_norm": 0.1,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "35",
+ "files": [
+ "edges.pkl",
+ "ll_model_510.pth",
+ "ll_model_cfg_510.pkl",
+ "meta_510.json"
+ ],
+ "task_description": "Alternates capitalization of each character in words.",
+ "vocab": [
+ "J",
+ "oCLrZaW",
+ "no",
+ "poiVg",
+ "V",
+ "b",
+ "LB",
+ "TPSI"
+ ],
+ "max_seq_len": 10,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 4,
+ "n_ctx": 10,
+ "d_head": 1,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 16,
+ "act_fn": "gelu",
+ "d_vocab": 10,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cpu",
+ "n_devices": 1,
+ "attention_dir": "causal",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.1539600717839002,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 8,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 384,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.001,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 0.4,
+ "epochs": 2000,
+ "act_fn": "gelu",
+ "clip_grad_norm": 0.1,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "36",
+ "files": [
+ "edges.pkl",
+ "ll_model_10110.pth",
+ "ll_model_cfg_10110.pkl",
+ "meta_10110.json"
+ ],
+ "task_description": "Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.",
+ "vocab": [
+ "\ud83d\ude22",
+ "\ud83d\udcd8",
+ "\ud83d\ude0a"
+ ],
+ "max_seq_len": 10,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 4,
+ "n_ctx": 10,
+ "d_head": 1,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 16,
+ "act_fn": "gelu",
+ "d_vocab": 5,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cuda",
+ "n_devices": 1,
+ "attention_dir": "causal",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.19402850002906638,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 3,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 384,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.001,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 10.0,
+ "epochs": 2000,
+ "act_fn": "gelu",
+ "clip_grad_norm": 0.1,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "37",
+ "files": [
+ "edges.pkl",
+ "ll_model_510.pth",
+ "ll_model_cfg_510.pkl",
+ "meta_510.json"
+ ],
+ "task_description": "Reverses each word in the sequence except for specified exclusions.",
+ "vocab": [
+ "J",
+ "oCLrZaW",
+ "no",
+ "poiVg",
+ "V",
+ "b",
+ "LB",
+ "TPSI"
+ ],
+ "max_seq_len": 10,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 4,
+ "n_ctx": 10,
+ "d_head": 1,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 16,
+ "act_fn": "gelu",
+ "d_vocab": 10,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cpu",
+ "n_devices": 1,
+ "attention_dir": "causal",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.1539600717839002,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 8,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 384,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.001,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 0.4,
+ "epochs": 2000,
+ "act_fn": "gelu",
+ "clip_grad_norm": 0.1,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "38",
+ "files": [
+ "edges.pkl",
+ "ll_model_510.pth",
+ "ll_model_cfg_510.pkl",
+ "meta_510.json"
+ ],
+ "task_description": "Checks if tokens alternate between two types.",
+ "vocab": [
+ "b",
+ "a",
+ "c"
+ ],
+ "max_seq_len": 10,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 20,
+ "n_ctx": 10,
+ "d_head": 5,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 80,
+ "act_fn": "gelu",
+ "d_vocab": 5,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cpu",
+ "n_devices": 1,
+ "attention_dir": "causal",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.1539600717839002,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 2,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 9600,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.001,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 0.4,
+ "epochs": 2000,
+ "act_fn": "gelu",
+ "clip_grad_norm": 0.1,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "4",
+ "files": [
+ "edges.pkl",
+ "ll_model_510.pth",
+ "ll_model_cfg_510.pkl",
+ "meta_510.json"
+ ],
+ "task_description": "Return fraction of previous open tokens minus the fraction of close tokens.",
+ "vocab": [
+ "b",
+ "(",
+ "c",
+ ")",
+ "a"
+ ],
+ "max_seq_len": 10,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 20,
+ "n_ctx": 10,
+ "d_head": 5,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 80,
+ "act_fn": "gelu",
+ "d_vocab": 7,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cpu",
+ "n_devices": 1,
+ "attention_dir": "causal",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.17056057308448835,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 1,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 9600,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.001,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 0.4,
+ "epochs": 2000,
+ "act_fn": "gelu",
+ "clip_grad_norm": 0.1,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "8",
+ "files": [
+ "edges.pkl",
+ "ll_model_510.pth",
+ "ll_model_cfg_510.pkl",
+ "meta_510.json"
+ ],
+ "task_description": "Fills gaps between tokens with a specified filler.",
+ "vocab": [
+ "J",
+ "oCLrZaW",
+ "no",
+ "poiVg",
+ "V",
+ "b",
+ "LB",
+ "TPSI"
+ ],
+ "max_seq_len": 10,
+ "min_seq_len": 4,
+ "transformer_cfg": {
+ "n_layers": 2,
+ "d_model": 20,
+ "n_ctx": 10,
+ "d_head": 5,
+ "model_name": "custom",
+ "n_heads": 4,
+ "d_mlp": 80,
+ "act_fn": "gelu",
+ "d_vocab": 10,
+ "eps": 1e-05,
+ "use_attn_result": true,
+ "use_attn_scale": true,
+ "use_split_qkv_input": true,
+ "use_hook_mlp_in": true,
+ "use_attn_in": false,
+ "use_local_attn": false,
+ "original_architecture": null,
+ "from_checkpoint": false,
+ "checkpoint_index": null,
+ "checkpoint_label_type": null,
+ "checkpoint_value": null,
+ "tokenizer_name": null,
+ "window_size": null,
+ "attn_types": null,
+ "init_mode": "gpt2",
+ "normalization_type": null,
+ "device": "cpu",
+ "n_devices": 1,
+ "attention_dir": "causal",
+ "attn_only": false,
+ "seed": 0,
+ "initializer_range": 0.13333333333333333,
+ "init_weights": true,
+ "scale_attn_by_inverse_layer_idx": false,
+ "positional_embedding_type": "standard",
+ "final_rms": false,
+ "d_vocab_out": 8,
+ "parallel_attn_mlp": false,
+ "rotary_dim": null,
+ "n_params": 9600,
+ "use_hook_tokens": false,
+ "gated_mlp": false,
+ "default_prepend_bos": true,
+ "dtype": "torch.float32",
+ "tokenizer_prepends_bos": null,
+ "n_key_value_heads": null,
+ "post_embedding_ln": false,
+ "rotary_base": 10000,
+ "trust_remote_code": false,
+ "rotary_adjacent_pairs": false
+ },
+ "training_args": {
+ "atol": 0.05,
+ "lr": 0.01,
+ "use_single_loss": false,
+ "iit_weight": 1.0,
+ "behavior_weight": 1.0,
+ "strict_weight": 0.4,
+ "epochs": 500,
+ "act_fn": "gelu",
+ "clip_grad_norm": 1.0,
+ "lr_scheduler": ""
+ }
+ },
+ {
+ "case_id": "ioi",
+ "files": [
+ "corr_100_100_40.json",
+ "ll_model_100_100_40.pth"
+ ]
+ },
+ {
+ "case_id": "ioi_next_token",
+ "files": [
+ "corr_100_100_40.json",
+ "ll_model_100_100_40.pth",
+ "training_args.json"
+ ]
+ }
+ ]
+}