Update metadata files

benchmark_cases_metadata.csv

@@ -1,19 +1,19 @@
-case_id,task_description,max_seq_len,min_seq_len,training_args.atol,training_args.lr,training_args.use_single_loss,training_args.iit_weight,training_args.behavior_weight,training_args.strict_weight,training_args.epochs,training_args.act_fn,training_args.clip_grad_norm,training_args.lr_scheduler,transformer_cfg.n_layers,transformer_cfg.d_model,transformer_cfg.n_ctx,transformer_cfg.d_head,transformer_cfg.model_name,transformer_cfg.n_heads,transformer_cfg.d_mlp,transformer_cfg.act_fn,transformer_cfg.d_vocab,transformer_cfg.eps,transformer_cfg.use_attn_result,transformer_cfg.use_attn_scale,transformer_cfg.use_split_qkv_input,transformer_cfg.use_hook_mlp_in,transformer_cfg.use_attn_in,transformer_cfg.use_local_attn,transformer_cfg.original_architecture,transformer_cfg.from_checkpoint,transformer_cfg.checkpoint_index,transformer_cfg.checkpoint_label_type,transformer_cfg.checkpoint_value,transformer_cfg.tokenizer_name,transformer_cfg.window_size,transformer_cfg.attn_types,transformer_cfg.init_mode,transformer_cfg.normalization_type,transformer_cfg.device,transformer_cfg.n_devices,transformer_cfg.attention_dir,transformer_cfg.attn_only,transformer_cfg.seed,transformer_cfg.initializer_range,transformer_cfg.init_weights,transformer_cfg.scale_attn_by_inverse_layer_idx,transformer_cfg.positional_embedding_type,transformer_cfg.final_rms,transformer_cfg.d_vocab_out,transformer_cfg.parallel_attn_mlp,transformer_cfg.rotary_dim,transformer_cfg.n_params,transformer_cfg.use_hook_tokens,transformer_cfg.gated_mlp,transformer_cfg.default_prepend_bos,transformer_cfg.dtype,transformer_cfg.tokenizer_prepends_bos,transformer_cfg.n_key_value_heads,transformer_cfg.post_embedding_ln,transformer_cfg.rotary_base,transformer_cfg.trust_remote_code,transformer_cfg.rotary_adjacent_pairs
-11,Counts the number of words in a sequence based on their length.,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,12.0,10.0,3.0,custom,4.0,48.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1460593486680443,True,False,standard,False,5.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
-13,"Analyzes the trend (increasing, decreasing, constant) of numeric tokens.",10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,bidirectional,False,0.0,0.1460593486680443,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
-18,"Classify each token based on its frequency as 'rare', 'common', or 'frequent'.",10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,12.0,10.0,3.0,custom,4.0,48.0,gelu,7.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,bidirectional,False,0.0,0.12344267996967354,True,False,standard,False,3.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
-19,Removes consecutive duplicate tokens from a sequence.,15.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,32.0,15.0,8.0,custom,4.0,128.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.15689290811054724,True,False,standard,False,3.0,False,,24576.0,False,False,True,torch.float32,,,False,10000.0,False,False
-20,Detect spam messages based on appearance of spam keywords.,10.0,4.0,0.05,0.001,False,1.0,1.0,1.0,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,14.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cuda,1.0,causal,False,0.0,0.16,True,False,standard,False,2.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
-21,Extract unique tokens from a string,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1885618083164127,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
-24,Identifies the first occurrence of each token in a sequence.,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1885618083164127,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
-3,Returns the fraction of 'x' in the input up to the i-th position for all i.,5.0,4.0,0.05,0.001,False,1.0,1.0,10.0,2000.0,gelu,0.1,,2.0,12.0,5.0,3.0,custom,4.0,48.0,gelu,6.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.22188007849009167,True,False,standard,False,1.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
-33,Checks if each token's length is odd or even.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.17457431218879393,True,False,standard,False,2.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
-34,Calculate the ratio of vowels to consonants in each word.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.16329931618554522,True,False,standard,False,5.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
-35,Alternates capitalization of each character in words.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,8.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
-36,"Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.",10.0,4.0,0.05,0.001,False,1.0,1.0,10.0,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cuda,1.0,causal,False,0.0,0.19402850002906638,True,False,standard,False,3.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
-37,Reverses each word in the sequence except for specified exclusions.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,8.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
-38,Checks if tokens alternate between two types.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,2.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
-4,Return fraction of previous open tokens minus the fraction of close tokens.,10.0,4.0,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,7.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.17056057308448835,True,False,standard,False,1.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
-8,Fills gaps between tokens with a specified filler.,10.0,4.0,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.13333333333333333,True,False,standard,False,8.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
-ioi,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-ioi_next_token,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+case_id,url,task_description,max_seq_len,min_seq_len,training_args.atol,training_args.lr,training_args.use_single_loss,training_args.iit_weight,training_args.behavior_weight,training_args.strict_weight,training_args.epochs,training_args.act_fn,training_args.clip_grad_norm,training_args.lr_scheduler,transformer_cfg.n_layers,transformer_cfg.d_model,transformer_cfg.n_ctx,transformer_cfg.d_head,transformer_cfg.model_name,transformer_cfg.n_heads,transformer_cfg.d_mlp,transformer_cfg.act_fn,transformer_cfg.d_vocab,transformer_cfg.eps,transformer_cfg.use_attn_result,transformer_cfg.use_attn_scale,transformer_cfg.use_split_qkv_input,transformer_cfg.use_hook_mlp_in,transformer_cfg.use_attn_in,transformer_cfg.use_local_attn,transformer_cfg.original_architecture,transformer_cfg.from_checkpoint,transformer_cfg.checkpoint_index,transformer_cfg.checkpoint_label_type,transformer_cfg.checkpoint_value,transformer_cfg.tokenizer_name,transformer_cfg.window_size,transformer_cfg.attn_types,transformer_cfg.init_mode,transformer_cfg.normalization_type,transformer_cfg.device,transformer_cfg.n_devices,transformer_cfg.attention_dir,transformer_cfg.attn_only,transformer_cfg.seed,transformer_cfg.initializer_range,transformer_cfg.init_weights,transformer_cfg.scale_attn_by_inverse_layer_idx,transformer_cfg.positional_embedding_type,transformer_cfg.final_rms,transformer_cfg.d_vocab_out,transformer_cfg.parallel_attn_mlp,transformer_cfg.rotary_dim,transformer_cfg.n_params,transformer_cfg.use_hook_tokens,transformer_cfg.gated_mlp,transformer_cfg.default_prepend_bos,transformer_cfg.dtype,transformer_cfg.tokenizer_prepends_bos,transformer_cfg.n_key_value_heads,transformer_cfg.post_embedding_ln,transformer_cfg.rotary_base,transformer_cfg.trust_remote_code,transformer_cfg.rotary_adjacent_pairs
+11,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/11,Counts the number of words in a sequence based on their length.,10,4,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,12.0,10.0,3.0,custom,4.0,48.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1460593486680443,True,False,standard,False,5.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
+13,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/13,"Analyzes the trend (increasing, decreasing, constant) of numeric tokens.",10,4,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,bidirectional,False,0.0,0.1460593486680443,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+18,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/18,"Classify each token based on its frequency as 'rare', 'common', or 'frequent'.",10,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,12.0,10.0,3.0,custom,4.0,48.0,gelu,7.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,bidirectional,False,0.0,0.12344267996967354,True,False,standard,False,3.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
+19,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/19,Removes consecutive duplicate tokens from a sequence.,15,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,32.0,15.0,8.0,custom,4.0,128.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.15689290811054724,True,False,standard,False,3.0,False,,24576.0,False,False,True,torch.float32,,,False,10000.0,False,False
+20,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/20,Detect spam messages based on appearance of spam keywords.,10,4,0.05,0.001,False,1.0,1.0,1.0,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,14.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cuda,1.0,causal,False,0.0,0.16,True,False,standard,False,2.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+21,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/21,Extract unique tokens from a string,10,4,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1885618083164127,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+24,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/24,Identifies the first occurrence of each token in a sequence.,10,4,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1885618083164127,True,False,standard,False,3.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+3,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/3,Returns the fraction of 'x' in the input up to the i-th position for all i.,5,4,0.05,0.001,False,1.0,1.0,10.0,2000.0,gelu,0.1,,2.0,12.0,5.0,3.0,custom,4.0,48.0,gelu,6.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.22188007849009167,True,False,standard,False,1.0,False,,3456.0,False,False,True,torch.float32,,,False,10000.0,False,False
+33,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/33,Checks if each token's length is odd or even.,10,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.17457431218879393,True,False,standard,False,2.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+34,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/34,Calculate the ratio of vowels to consonants in each word.,10,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.16329931618554522,True,False,standard,False,5.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+35,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/35,Alternates capitalization of each character in words.,10,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,8.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+36,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/36,"Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.",10,4,0.05,0.001,False,1.0,1.0,10.0,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cuda,1.0,causal,False,0.0,0.19402850002906638,True,False,standard,False,3.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+37,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/37,Reverses each word in the sequence except for specified exclusions.,10,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,4.0,10.0,1.0,custom,4.0,16.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,8.0,False,,384.0,False,False,True,torch.float32,,,False,10000.0,False,False
+38,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/38,Checks if tokens alternate between two types.,10,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,5.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.1539600717839002,True,False,standard,False,2.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+4,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/4,Return fraction of previous open tokens minus the fraction of close tokens.,10,4,0.05,0.001,False,1.0,1.0,0.4,2000.0,gelu,0.1,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,7.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.17056057308448835,True,False,standard,False,1.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+8,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/8,Fills gaps between tokens with a specified filler.,10,4,0.05,0.01,False,1.0,1.0,0.4,500.0,gelu,1.0,,2.0,20.0,10.0,5.0,custom,4.0,80.0,gelu,10.0,1e-05,True,True,True,True,False,False,,False,,,,,,,gpt2,,cpu,1.0,causal,False,0.0,0.13333333333333333,True,False,standard,False,8.0,False,,9600.0,False,False,True,torch.float32,,,False,10000.0,False,False
+ioi,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi,Indirect object identification,16,16,,,True,,,,,,,,,,,,,,,,,,True,True,True,True,True,True,,True,,,,,,,,,,,,True,,,True,True,,True,,True,,,True,True,True,,,,True,,True,True
+ioi_next_token,https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi_next_token,Indirect object identification,16,16,,,True,,,,,,,,,,,,,,,,,,True,True,True,True,True,True,,True,,,,,,,,,,,,True,,,True,True,,True,,True,,,True,True,True,,,,True,,True,True

benchmark_cases_metadata.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01f473326ab84033784efd049808e3f0395bf577e23311a60f9b05eff32f5ca3
-size 49942
+oid sha256:12652b82dbded2521f44e1219ade14c88e6bd787db5a2141803db257fb375e87
+size 51034

benchmark_metadata.json

@@ -2,28 +2,43 @@
  "name": "InterpBench",
  "version": "1.0.0",
  "description": "A benchmark of transformers with known circuits for evaluating mechanistic interpretability techniques.",
+ "license": "https://creativecommons.org/licenses/by/4.0/",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench",
  "cases": [
  {
  "case_id": "11",
- "files": [
- "edges.pkl",
- "ll_model_510.pth",
- "ll_model_cfg_510.pkl",
- "meta_510.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/11",
  "task_description": "Counts the number of words in a sequence based on their length.",
  "vocab": [
  "J",
- "oCLrZaW",
- "no",
- "poiVg",
+ "LB",
+ "TPSI",
  "V",
  "b",
- "LB",
- "TPSI"
+ "no",
+ "oCLrZaW",
+ "poiVg"
  ],
  "max_seq_len": 10,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_510.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model_510.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_510.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model_cfg_510.pkl"
+ },
+ {
+ "file_name": "meta_510.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/meta_510.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 12,
@@ -91,12 +106,7 @@
  },
  {
  "case_id": "13",
- "files": [
- "edges.pkl",
- "ll_model_510.pth",
- "ll_model_cfg_510.pkl",
- "meta_510.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/13",
  "task_description": "Analyzes the trend (increasing, decreasing, constant) of numeric tokens.",
  "vocab": [
  0,
@@ -105,6 +115,24 @@
  ],
  "max_seq_len": 10,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_510.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model_510.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_510.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model_cfg_510.pkl"
+ },
+ {
+ "file_name": "meta_510.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/meta_510.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 20,
@@ -172,22 +200,35 @@
  },
  {
  "case_id": "18",
- "files": [
- "edges.pkl",
- "ll_model_510.pth",
- "ll_model_cfg_510.pkl",
- "meta_510.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/18",
  "task_description": "Classify each token based on its frequency as 'rare', 'common', or 'frequent'.",
  "vocab": [
- "c",
- "e",
+ "a",
  "b",
+ "c",
  "d",
- "a"
+ "e"
  ],
  "max_seq_len": 10,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_510.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model_510.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_510.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model_cfg_510.pkl"
+ },
+ {
+ "file_name": "meta_510.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/meta_510.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 12,
@@ -255,20 +296,33 @@
  },
  {
  "case_id": "19",
- "files": [
- "edges.pkl",
- "ll_model_510.pth",
- "ll_model_cfg_510.pkl",
- "meta_510.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/19",
  "task_description": "Removes consecutive duplicate tokens from a sequence.",
  "vocab": [
- "b",
  "a",
+ "b",
  "c"
  ],
  "max_seq_len": 15,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_510.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model_510.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_510.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model_cfg_510.pkl"
+ },
+ {
+ "file_name": "meta_510.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/meta_510.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 32,
@@ -336,29 +390,42 @@
  },
  {
  "case_id": "20",
- "files": [
- "edges.pkl",
- "ll_model_1110.pth",
- "ll_model_cfg_1110.pkl",
- "meta_1110.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/20",
  "task_description": "Detect spam messages based on appearance of spam keywords.",
  "vocab": [
  "J",
- "spam",
- "offer",
- "click",
- "oCLrZaW",
- "no",
- "poiVg",
+ "LB",
+ "TPSI",
  "V",
  "b",
- "LB",
+ "click",
+ "no",
  "now",
- "TPSI"
+ "oCLrZaW",
+ "offer",
+ "poiVg",
+ "spam"
  ],
  "max_seq_len": 10,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_1110.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model_1110.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_1110.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model_cfg_1110.pkl"
+ },
+ {
+ "file_name": "meta_1110.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/meta_1110.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 4,
@@ -426,20 +493,33 @@
  },
  {
  "case_id": "21",
- "files": [
- "edges.pkl",
- "ll_model_510.pth",
- "ll_model_cfg_510.pkl",
- "meta_510.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/21",
  "task_description": "Extract unique tokens from a string",
  "vocab": [
- "b",
  "a",
+ "b",
  "c"
  ],
  "max_seq_len": 10,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_510.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model_510.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_510.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model_cfg_510.pkl"
+ },
+ {
+ "file_name": "meta_510.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/meta_510.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 20,
@@ -507,20 +587,33 @@
  },
  {
  "case_id": "24",
- "files": [
- "edges.pkl",
- "ll_model_510.pth",
- "ll_model_cfg_510.pkl",
- "meta_510.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/24",
  "task_description": "Identifies the first occurrence of each token in a sequence.",
  "vocab": [
- "b",
  "a",
+ "b",
  "c"
  ],
  "max_seq_len": 10,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/24/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_510.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/24/ll_model_510.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_510.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/24/ll_model_cfg_510.pkl"
+ },
+ {
+ "file_name": "meta_510.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/24/meta_510.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 20,
@@ -588,21 +681,34 @@
  },
  {
  "case_id": "3",
- "files": [
- "edges.pkl",
- "ll_model_10110.pth",
- "ll_model_cfg_10110.pkl",
- "meta_10110.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/3",
  "task_description": "Returns the fraction of 'x' in the input up to the i-th position for all i.",
  "vocab": [
- "x",
- "b",
  "a",
- "c"
+ "b",
+ "c",
+ "x"
  ],
  "max_seq_len": 5,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_10110.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model_10110.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_10110.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model_cfg_10110.pkl"
+ },
+ {
+ "file_name": "meta_10110.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/meta_10110.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 12,
@@ -670,25 +776,38 @@
  },
  {
  "case_id": "33",
- "files": [
- "edges.pkl",
- "ll_model_510.pth",
- "ll_model_cfg_510.pkl",
- "meta_510.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/33",
  "task_description": "Checks if each token's length is odd or even.",
  "vocab": [
  "J",
- "oCLrZaW",
- "no",
- "poiVg",
+ "LB",
+ "TPSI",
  "V",
  "b",
- "LB",
- "TPSI"
+ "no",
+ "oCLrZaW",
+ "poiVg"
  ],
  "max_seq_len": 10,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_510.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model_510.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_510.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model_cfg_510.pkl"
+ },
+ {
+ "file_name": "meta_510.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/meta_510.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 4,
@@ -756,25 +875,38 @@
  },
  {
  "case_id": "34",
- "files": [
- "edges.pkl",
- "ll_model_510.pth",
- "ll_model_cfg_510.pkl",
- "meta_510.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/34",
  "task_description": "Calculate the ratio of vowels to consonants in each word.",
  "vocab": [
  "J",
- "oCLrZaW",
- "no",
- "poiVg",
+ "LB",
+ "TPSI",
  "V",
  "b",
- "LB",
- "TPSI"
+ "no",
+ "oCLrZaW",
+ "poiVg"
  ],
  "max_seq_len": 10,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_510.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model_510.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_510.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model_cfg_510.pkl"
+ },
+ {
+ "file_name": "meta_510.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/meta_510.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 4,
@@ -842,25 +974,38 @@
  },
  {
  "case_id": "35",
- "files": [
- "edges.pkl",
- "ll_model_510.pth",
- "ll_model_cfg_510.pkl",
- "meta_510.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/35",
  "task_description": "Alternates capitalization of each character in words.",
  "vocab": [
  "J",
- "oCLrZaW",
- "no",
- "poiVg",
+ "LB",
+ "TPSI",
  "V",
  "b",
- "LB",
- "TPSI"
+ "no",
+ "oCLrZaW",
+ "poiVg"
  ],
  "max_seq_len": 10,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_510.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model_510.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_510.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model_cfg_510.pkl"
+ },
+ {
+ "file_name": "meta_510.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/meta_510.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 4,
@@ -928,20 +1073,33 @@
  },
  {
  "case_id": "36",
- "files": [
- "edges.pkl",
- "ll_model_10110.pth",
- "ll_model_cfg_10110.pkl",
- "meta_10110.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/36",
  "task_description": "Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.",
  "vocab": [
- "\ud83d\ude22",
  "\ud83d\udcd8",
- "\ud83d\ude0a"
+ "\ud83d\ude0a",
+ "\ud83d\ude22"
  ],
  "max_seq_len": 10,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_10110.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model_10110.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_10110.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model_cfg_10110.pkl"
+ },
+ {
+ "file_name": "meta_10110.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/meta_10110.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 4,
@@ -1009,25 +1167,38 @@
  },
  {
  "case_id": "37",
- "files": [
- "edges.pkl",
- "ll_model_510.pth",
- "ll_model_cfg_510.pkl",
- "meta_510.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/37",
  "task_description": "Reverses each word in the sequence except for specified exclusions.",
  "vocab": [
  "J",
- "oCLrZaW",
- "no",
- "poiVg",
+ "LB",
+ "TPSI",
  "V",
  "b",
- "LB",
- "TPSI"
+ "no",
+ "oCLrZaW",
+ "poiVg"
  ],
  "max_seq_len": 10,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_510.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model_510.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_510.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model_cfg_510.pkl"
+ },
+ {
+ "file_name": "meta_510.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/meta_510.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 4,
@@ -1095,20 +1266,33 @@
  },
  {
  "case_id": "38",
- "files": [
- "edges.pkl",
- "ll_model_510.pth",
- "ll_model_cfg_510.pkl",
- "meta_510.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/38",
  "task_description": "Checks if tokens alternate between two types.",
  "vocab": [
- "b",
  "a",
+ "b",
  "c"
  ],
  "max_seq_len": 10,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/38/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_510.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/38/ll_model_510.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_510.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/38/ll_model_cfg_510.pkl"
+ },
+ {
+ "file_name": "meta_510.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/38/meta_510.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 20,
@@ -1176,22 +1360,35 @@
  },
  {
  "case_id": "4",
- "files": [
- "edges.pkl",
- "ll_model_510.pth",
- "ll_model_cfg_510.pkl",
- "meta_510.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/4",
  "task_description": "Return fraction of previous open tokens minus the fraction of close tokens.",
  "vocab": [
- "b",
  "(",
- "c",
  ")",
- "a"
+ "a",
+ "b",
+ "c"
  ],
  "max_seq_len": 10,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_510.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model_510.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_510.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model_cfg_510.pkl"
+ },
+ {
+ "file_name": "meta_510.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/meta_510.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 20,
@@ -1259,25 +1456,38 @@
  },
  {
  "case_id": "8",
- "files": [
- "edges.pkl",
- "ll_model_510.pth",
- "ll_model_cfg_510.pkl",
- "meta_510.json"
- ],
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/8",
  "task_description": "Fills gaps between tokens with a specified filler.",
  "vocab": [
  "J",
- "oCLrZaW",
- "no",
- "poiVg",
+ "LB",
+ "TPSI",
  "V",
  "b",
- "LB",
- "TPSI"
+ "no",
+ "oCLrZaW",
+ "poiVg"
  ],
  "max_seq_len": 10,
  "min_seq_len": 4,
+ "files": [
+ {
+ "file_name": "edges.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/8/edges.pkl"
+ },
+ {
+ "file_name": "ll_model_510.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/8/ll_model_510.pth"
+ },
+ {
+ "file_name": "ll_model_cfg_510.pkl",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/8/ll_model_cfg_510.pkl"
+ },
+ {
+ "file_name": "meta_510.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/8/meta_510.json"
+ }
+ ],
  "transformer_cfg": {
  "n_layers": 2,
  "d_model": 20,
@@ -1345,17 +1555,40 @@
  },
  {
  "case_id": "ioi",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi",
+ "task_description": "Indirect object identification",
+ "max_seq_len": 16,
+ "min_seq_len": 16,
  "files": [
- "corr_100_100_40.json",
- "ll_model_100_100_40.pth"
+ {
+ "file_name": "corr_100_100_40.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/corr_100_100_40.json"
+ },
+ {
+ "file_name": "ll_model_100_100_40.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model_100_100_40.pth"
+ }
  ]
  },
  {
  "case_id": "ioi_next_token",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi_next_token",
+ "task_description": "Indirect object identification",
+ "max_seq_len": 16,
+ "min_seq_len": 16,
  "files": [
- "corr_100_100_40.json",
- "ll_model_100_100_40.pth",
- "training_args.json"
+ {
+ "file_name": "corr_100_100_40.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/corr_100_100_40.json"
+ },
+ {
+ "file_name": "ll_model_100_100_40.pth",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model_100_100_40.pth"
+ },
+ {
+ "file_name": "training_args.json",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/training_args.json"
+ }
  ]
  }
  ]

benchmark_metadata_croissant.json

@@ -0,0 +1,1103 @@
+{
+ "@context": {
+ "@language": "en",
+ "@vocab": "https://schema.org/",
+ "citeAs": "cr:citeAs",
+ "column": "cr:column",
+ "conformsTo": "dct:conformsTo",
+ "cr": "http://mlcommons.org/croissant/",
+ "rai": "http://mlcommons.org/croissant/RAI/",
+ "data": {
+ "@id": "cr:data",
+ "@type": "@json"
+ },
+ "dataType": {
+ "@id": "cr:dataType",
+ "@type": "@vocab"
+ },
+ "dct": "http://purl.org/dc/terms/",
+ "examples": {
+ "@id": "cr:examples",
+ "@type": "@json"
+ },
+ "extract": "cr:extract",
+ "field": "cr:field",
+ "fileProperty": "cr:fileProperty",
+ "fileObject": "cr:fileObject",
+ "fileSet": "cr:fileSet",
+ "format": "cr:format",
+ "includes": "cr:includes",
+ "isLiveDataset": "cr:isLiveDataset",
+ "jsonPath": "cr:jsonPath",
+ "key": "cr:key",
+ "md5": "cr:md5",
+ "parentField": "cr:parentField",
+ "path": "cr:path",
+ "recordSet": "cr:recordSet",
+ "references": "cr:references",
+ "regex": "cr:regex",
+ "repeated": "cr:repeated",
+ "replace": "cr:replace",
+ "sc": "https://schema.org/",
+ "separator": "cr:separator",
+ "source": "cr:source",
+ "subField": "cr:subField",
+ "transform": "cr:transform"
+ },
+ "@type": "sc:Dataset",
+ "name": "InterpBench",
+ "description": "A benchmark of transformers with known circuits for evaluating mechanistic interpretability techniques.",
+ "conformsTo": "http://mlcommons.org/croissant/1.0",
+ "license": "https://creativecommons.org/licenses/by/4.0/",
+ "url": "https://huggingface.co/cybershiptrooper/InterpBench",
+ "version": "1.0.0",
+ "distribution": [
+ {
+ "@type": "cr:FileObject",
+ "@id": "hf-repository",
+ "name": "hf-repository",
+ "description": "The Hugging Face git repository.",
+ "contentUrl": "https://huggingface.co/cybershiptrooper/InterpBench",
+ "encodingFormat": "git+https",
+ "sha256": "main"
+ },
+ {
+ "@type": "cr:FileObject",
+ "@id": "benchmark-cases-parquet",
+ "name": "benchmark-cases-parquet",
+ "description": "Parquet file describing all the cases in the benchmark.",
+ "containedIn": {
+ "@id": "hf-repository"
+ },
+ "encodingFormat": "application/x-parquet"
+ },
+ {
+ "@type": "cr:FileSet",
+ "@id": "training-args",
+ "name": "training-args",
+ "description": "Training arguments.",
+ "containedIn": {
+ "@id": "hf-repository"
+ },
+ "encodingFormat": "application/json",
+ "includes": "*/meta_[0-9]*.json"
+ },
+ {
+ "@type": "cr:FileSet",
+ "@id": "circuits",
+ "name": "circuits",
+ "description": "Ground truth circuits (Pickle).",
+ "containedIn": {
+ "@id": "hf-repository"
+ },
+ "encodingFormat": "application/octet-stream",
+ "includes": "*/edges.pkl"
+ },
+ {
+ "@type": "cr:FileSet",
+ "@id": "weights",
+ "name": "weights",
+ "description": "Serialized PyTorch state dictionaries (Pickle).",
+ "containedIn": {
+ "@id": "hf-repository"
+ },
+ "encodingFormat": "application/octet-stream",
+ "includes": "*/ll_model_[0-9]*.pkl"
+ },
+ {
+ "@type": "cr:FileSet",
+ "@id": "cfgs",
+ "name": "cfgs",
+ "description": "Architecture configs (Pickle).",
+ "containedIn": {
+ "@id": "hf-repository"
+ },
+ "encodingFormat": "application/octet-stream",
+ "includes": "*/ll_model_cfg_[0-9]*.pkl"
+ }
+ ],
+ "recordSet": [
+ {
+ "@type": "cr:RecordSet",
+ "@id": "cases",
+ "name": "cases",
+ "field": [
+ {
+ "@type": "cr:Field",
+ "@id": "case_id",
+ "name": "case_id",
+ "description": "Column 'case_id' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Text",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "case_id"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "url",
+ "name": "url",
+ "description": "Column 'url' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Text",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "url"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "task_description",
+ "name": "task_description",
+ "description": "Column 'task_description' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Text",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "task_description"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "max_seq_len",
+ "name": "max_seq_len",
+ "description": "Column 'max_seq_len' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Integer",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "max_seq_len"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "min_seq_len",
+ "name": "min_seq_len",
+ "description": "Column 'min_seq_len' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Integer",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "min_seq_len"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "training_args.atol",
+ "name": "training_args.atol",
+ "description": "Column 'training_args.atol' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "training_args.atol"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "training_args.lr",
+ "name": "training_args.lr",
+ "description": "Column 'training_args.lr' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "training_args.lr"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "training_args.use_single_loss",
+ "name": "training_args.use_single_loss",
+ "description": "Column 'training_args.use_single_loss' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "training_args.use_single_loss"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "training_args.iit_weight",
+ "name": "training_args.iit_weight",
+ "description": "Column 'training_args.iit_weight' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "training_args.iit_weight"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "training_args.behavior_weight",
+ "name": "training_args.behavior_weight",
+ "description": "Column 'training_args.behavior_weight' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "training_args.behavior_weight"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "training_args.strict_weight",
+ "name": "training_args.strict_weight",
+ "description": "Column 'training_args.strict_weight' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "training_args.strict_weight"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "training_args.epochs",
+ "name": "training_args.epochs",
+ "description": "Column 'training_args.epochs' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "training_args.epochs"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "training_args.act_fn",
+ "name": "training_args.act_fn",
+ "description": "Column 'training_args.act_fn' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Text",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "training_args.act_fn"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "training_args.clip_grad_norm",
+ "name": "training_args.clip_grad_norm",
+ "description": "Column 'training_args.clip_grad_norm' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "training_args.clip_grad_norm"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "training_args.lr_scheduler",
+ "name": "training_args.lr_scheduler",
+ "description": "Column 'training_args.lr_scheduler' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Text",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "training_args.lr_scheduler"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.n_layers",
+ "name": "transformer_cfg.n_layers",
+ "description": "Column 'transformer_cfg.n_layers' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.n_layers"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.d_model",
+ "name": "transformer_cfg.d_model",
+ "description": "Column 'transformer_cfg.d_model' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.d_model"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.n_ctx",
+ "name": "transformer_cfg.n_ctx",
+ "description": "Column 'transformer_cfg.n_ctx' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.n_ctx"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.d_head",
+ "name": "transformer_cfg.d_head",
+ "description": "Column 'transformer_cfg.d_head' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.d_head"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.model_name",
+ "name": "transformer_cfg.model_name",
+ "description": "Column 'transformer_cfg.model_name' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Text",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.model_name"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.n_heads",
+ "name": "transformer_cfg.n_heads",
+ "description": "Column 'transformer_cfg.n_heads' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.n_heads"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.d_mlp",
+ "name": "transformer_cfg.d_mlp",
+ "description": "Column 'transformer_cfg.d_mlp' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.d_mlp"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.act_fn",
+ "name": "transformer_cfg.act_fn",
+ "description": "Column 'transformer_cfg.act_fn' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Text",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.act_fn"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.d_vocab",
+ "name": "transformer_cfg.d_vocab",
+ "description": "Column 'transformer_cfg.d_vocab' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.d_vocab"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.eps",
+ "name": "transformer_cfg.eps",
+ "description": "Column 'transformer_cfg.eps' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.eps"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.use_attn_result",
+ "name": "transformer_cfg.use_attn_result",
+ "description": "Column 'transformer_cfg.use_attn_result' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.use_attn_result"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.use_attn_scale",
+ "name": "transformer_cfg.use_attn_scale",
+ "description": "Column 'transformer_cfg.use_attn_scale' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.use_attn_scale"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.use_split_qkv_input",
+ "name": "transformer_cfg.use_split_qkv_input",
+ "description": "Column 'transformer_cfg.use_split_qkv_input' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.use_split_qkv_input"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.use_hook_mlp_in",
+ "name": "transformer_cfg.use_hook_mlp_in",
+ "description": "Column 'transformer_cfg.use_hook_mlp_in' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.use_hook_mlp_in"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.use_attn_in",
+ "name": "transformer_cfg.use_attn_in",
+ "description": "Column 'transformer_cfg.use_attn_in' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.use_attn_in"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.use_local_attn",
+ "name": "transformer_cfg.use_local_attn",
+ "description": "Column 'transformer_cfg.use_local_attn' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.use_local_attn"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.original_architecture",
+ "name": "transformer_cfg.original_architecture",
+ "description": "Column 'transformer_cfg.original_architecture' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.original_architecture"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.from_checkpoint",
+ "name": "transformer_cfg.from_checkpoint",
+ "description": "Column 'transformer_cfg.from_checkpoint' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.from_checkpoint"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.checkpoint_index",
+ "name": "transformer_cfg.checkpoint_index",
+ "description": "Column 'transformer_cfg.checkpoint_index' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.checkpoint_index"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.checkpoint_label_type",
+ "name": "transformer_cfg.checkpoint_label_type",
+ "description": "Column 'transformer_cfg.checkpoint_label_type' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.checkpoint_label_type"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.checkpoint_value",
+ "name": "transformer_cfg.checkpoint_value",
+ "description": "Column 'transformer_cfg.checkpoint_value' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.checkpoint_value"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.tokenizer_name",
+ "name": "transformer_cfg.tokenizer_name",
+ "description": "Column 'transformer_cfg.tokenizer_name' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.tokenizer_name"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.window_size",
+ "name": "transformer_cfg.window_size",
+ "description": "Column 'transformer_cfg.window_size' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.window_size"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.attn_types",
+ "name": "transformer_cfg.attn_types",
+ "description": "Column 'transformer_cfg.attn_types' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.attn_types"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.init_mode",
+ "name": "transformer_cfg.init_mode",
+ "description": "Column 'transformer_cfg.init_mode' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Text",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.init_mode"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.normalization_type",
+ "name": "transformer_cfg.normalization_type",
+ "description": "Column 'transformer_cfg.normalization_type' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.normalization_type"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.device",
+ "name": "transformer_cfg.device",
+ "description": "Column 'transformer_cfg.device' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Text",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.device"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.n_devices",
+ "name": "transformer_cfg.n_devices",
+ "description": "Column 'transformer_cfg.n_devices' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.n_devices"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.attention_dir",
+ "name": "transformer_cfg.attention_dir",
+ "description": "Column 'transformer_cfg.attention_dir' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Text",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.attention_dir"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.attn_only",
+ "name": "transformer_cfg.attn_only",
+ "description": "Column 'transformer_cfg.attn_only' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.attn_only"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.seed",
+ "name": "transformer_cfg.seed",
+ "description": "Column 'transformer_cfg.seed' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.seed"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.initializer_range",
+ "name": "transformer_cfg.initializer_range",
+ "description": "Column 'transformer_cfg.initializer_range' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.initializer_range"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.init_weights",
+ "name": "transformer_cfg.init_weights",
+ "description": "Column 'transformer_cfg.init_weights' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.init_weights"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.scale_attn_by_inverse_layer_idx",
+ "name": "transformer_cfg.scale_attn_by_inverse_layer_idx",
+ "description": "Column 'transformer_cfg.scale_attn_by_inverse_layer_idx' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.scale_attn_by_inverse_layer_idx"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.positional_embedding_type",
+ "name": "transformer_cfg.positional_embedding_type",
+ "description": "Column 'transformer_cfg.positional_embedding_type' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Text",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.positional_embedding_type"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.final_rms",
+ "name": "transformer_cfg.final_rms",
+ "description": "Column 'transformer_cfg.final_rms' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.final_rms"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.d_vocab_out",
+ "name": "transformer_cfg.d_vocab_out",
+ "description": "Column 'transformer_cfg.d_vocab_out' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.d_vocab_out"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.parallel_attn_mlp",
+ "name": "transformer_cfg.parallel_attn_mlp",
+ "description": "Column 'transformer_cfg.parallel_attn_mlp' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.parallel_attn_mlp"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.rotary_dim",
+ "name": "transformer_cfg.rotary_dim",
+ "description": "Column 'transformer_cfg.rotary_dim' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.rotary_dim"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.n_params",
+ "name": "transformer_cfg.n_params",
+ "description": "Column 'transformer_cfg.n_params' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.n_params"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.use_hook_tokens",
+ "name": "transformer_cfg.use_hook_tokens",
+ "description": "Column 'transformer_cfg.use_hook_tokens' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.use_hook_tokens"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.gated_mlp",
+ "name": "transformer_cfg.gated_mlp",
+ "description": "Column 'transformer_cfg.gated_mlp' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.gated_mlp"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.default_prepend_bos",
+ "name": "transformer_cfg.default_prepend_bos",
+ "description": "Column 'transformer_cfg.default_prepend_bos' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.default_prepend_bos"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.dtype",
+ "name": "transformer_cfg.dtype",
+ "description": "Column 'transformer_cfg.dtype' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Text",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.dtype"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.tokenizer_prepends_bos",
+ "name": "transformer_cfg.tokenizer_prepends_bos",
+ "description": "Column 'transformer_cfg.tokenizer_prepends_bos' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.tokenizer_prepends_bos"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.n_key_value_heads",
+ "name": "transformer_cfg.n_key_value_heads",
+ "description": "Column 'transformer_cfg.n_key_value_heads' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.n_key_value_heads"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.post_embedding_ln",
+ "name": "transformer_cfg.post_embedding_ln",
+ "description": "Column 'transformer_cfg.post_embedding_ln' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.post_embedding_ln"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.rotary_base",
+ "name": "transformer_cfg.rotary_base",
+ "description": "Column 'transformer_cfg.rotary_base' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Float",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.rotary_base"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.trust_remote_code",
+ "name": "transformer_cfg.trust_remote_code",
+ "description": "Column 'transformer_cfg.trust_remote_code' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.trust_remote_code"
+ }
+ }
+ },
+ {
+ "@type": "cr:Field",
+ "@id": "transformer_cfg.rotary_adjacent_pairs",
+ "name": "transformer_cfg.rotary_adjacent_pairs",
+ "description": "Column 'transformer_cfg.rotary_adjacent_pairs' from the parquet file describing all the cases in the benchmark.",
+ "dataType": "sc:Boolean",
+ "source": {
+ "fileSet": {
+ "@id": "benchmark-cases-parquet"
+ },
+ "extract": {
+ "column": "transformer_cfg.rotary_adjacent_pairs"
+ }
+ }
+ }
+ ]
+ }
+ ]
+}