Add files using upload-large-folder tool

.gitattributes

@@ -38,3 +38,9 @@ random_seed_eval_results/autointerp/temp_random_seeds_google_gemma-2-2b_top_k_re
 random_seed_eval_results/autointerp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_3_custom_sae_eval_results.json filter=lfs diff=lfs merge=lfs -text
 random_seed_eval_results/autointerp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_4_custom_sae_eval_results.json filter=lfs diff=lfs merge=lfs -text
 random_seed_eval_results/autointerp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_2_custom_sae_eval_results.json filter=lfs diff=lfs merge=lfs -text
+old_relu_eval_results/autointerp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_3_custom_sae_eval_results.json filter=lfs diff=lfs merge=lfs -text
+old_relu_eval_results/autointerp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_0_custom_sae_eval_results.json filter=lfs diff=lfs merge=lfs -text
+old_relu_eval_results/autointerp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_1_custom_sae_eval_results.json filter=lfs diff=lfs merge=lfs -text
+old_relu_eval_results/autointerp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_5_custom_sae_eval_results.json filter=lfs diff=lfs merge=lfs -text
+old_relu_eval_results/autointerp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_2_custom_sae_eval_results.json filter=lfs diff=lfs merge=lfs -text
+old_relu_eval_results/autointerp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_4_custom_sae_eval_results.json filter=lfs diff=lfs merge=lfs -text

old_relu_eval_results/absorption/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_0_custom_sae_eval_results.json

@@ -0,0 +1,268 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50
+  },
+  "eval_id": "431e48c6-55b1-4fe2-a188-fdf282380ea2",
+  "datetime_epoch_millis": 1738783485930,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.09751356079257899,
+      "mean_full_absorption_score": 0.004661694890776431,
+      "mean_num_split_features": 1.0384615384615385,
+      "std_dev_absorption_fraction_score": 0.10468056684076392,
+      "std_dev_full_absorption_score": 0.00923750174821687,
+      "std_dev_num_split_features": 0.19611613513818404
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.10952248859777117,
+      "full_absorption_rate": 0.0007757951900698216,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 2578,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.037033530481398055,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 1664,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.32412380384249956,
+      "full_absorption_rate": 0.012851684612712747,
+      "num_full_absorption": 37,
+      "num_probe_true_positives": 2879,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.2652438933350776,
+      "full_absorption_rate": 0.015439429928741092,
+      "num_full_absorption": 26,
+      "num_probe_true_positives": 1684,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.029511883739283127,
+      "full_absorption_rate": 0.005829015544041451,
+      "num_full_absorption": 9,
+      "num_probe_true_positives": 1544,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.03661148893601744,
+      "full_absorption_rate": 0.0008598452278589854,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 1163,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.07005964729193298,
+      "full_absorption_rate": 0.0027223230490018148,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 1102,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.025913905709330304,
+      "full_absorption_rate": 0.0019821605550049554,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 1009,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.16489142397701123,
+      "full_absorption_rate": 0.005441354292623942,
+      "num_full_absorption": 9,
+      "num_probe_true_positives": 1654,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.017172634110049358,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 406,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.016921317745917687,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 665,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.043985977343370744,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 1195,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.02707755806490339,
+      "full_absorption_rate": 0.0011025358324145535,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 1814,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.04764539422889375,
+      "full_absorption_rate": 0.003778337531486146,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 794,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.11199046697502817,
+      "full_absorption_rate": 0.002857142857142857,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 1050,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.2957251949612795,
+      "full_absorption_rate": 0.002926421404682274,
+      "num_full_absorption": 7,
+      "num_probe_true_positives": 2392,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.00767947180411824,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 174,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.06558605221479297,
+      "full_absorption_rate": 0.0006016847172081829,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 1662,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.24032658294094394,
+      "full_absorption_rate": 0.015951790145338533,
+      "num_full_absorption": 45,
+      "num_probe_true_positives": 2821,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.04682942316731627,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 1677,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.33437220839551257,
+      "full_absorption_rate": 0.043824701195219126,
+      "num_full_absorption": 33,
+      "num_probe_true_positives": 753,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.017881854389667527,
+      "full_absorption_rate": 0.0011876484560570072,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 842,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.03906620984144505,
+      "full_absorption_rate": 0.0030721966205837174,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 651,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.11287436426111006,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 107,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.030323895956477616,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 193,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.016981908295905283,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 239,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_0",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/absorption/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_1_custom_sae_eval_results.json

@@ -0,0 +1,268 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50
+  },
+  "eval_id": "2f7686cc-05f0-4e61-bd5d-1e2c4da21721",
+  "datetime_epoch_millis": 1738785879962,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.12074450732711571,
+      "mean_full_absorption_score": 0.013239901484226383,
+      "mean_num_split_features": 1.1153846153846154,
+      "std_dev_absorption_fraction_score": 0.1336485927173351,
+      "std_dev_full_absorption_score": 0.026396743147366758,
+      "std_dev_num_split_features": 0.3258125936084211
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.1828675228579153,
+      "full_absorption_rate": 0.004266873545384018,
+      "num_full_absorption": 11,
+      "num_probe_true_positives": 2578,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.06300828457462988,
+      "full_absorption_rate": 0.003605769230769231,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 1664,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.37049847095340266,
+      "full_absorption_rate": 0.04307051059395624,
+      "num_full_absorption": 124,
+      "num_probe_true_positives": 2879,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.4320868702385805,
+      "full_absorption_rate": 0.083729216152019,
+      "num_full_absorption": 141,
+      "num_probe_true_positives": 1684,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.04056307305862558,
+      "full_absorption_rate": 0.007772020725388601,
+      "num_full_absorption": 12,
+      "num_probe_true_positives": 1544,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.0644915910448518,
+      "full_absorption_rate": 0.0025795356835769563,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 1163,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.06433309126945641,
+      "full_absorption_rate": 0.0027223230490018148,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 1102,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.04143727370568341,
+      "full_absorption_rate": 0.0019821605550049554,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 1009,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.18541141915011472,
+      "full_absorption_rate": 0.009673518742442563,
+      "num_full_absorption": 16,
+      "num_probe_true_positives": 1654,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.0066235494743313895,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 406,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.010681486258684182,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 665,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.09145893836903148,
+      "full_absorption_rate": 0.00502092050209205,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 1195,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.06668737220718245,
+      "full_absorption_rate": 0.004410143329658214,
+      "num_full_absorption": 8,
+      "num_probe_true_positives": 1814,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.0602452762340166,
+      "full_absorption_rate": 0.003778337531486146,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 794,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.12143825786919218,
+      "full_absorption_rate": 0.009523809523809525,
+      "num_full_absorption": 10,
+      "num_probe_true_positives": 1050,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.37822836353215283,
+      "full_absorption_rate": 0.022993311036789296,
+      "num_full_absorption": 55,
+      "num_probe_true_positives": 2392,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.006096479671690135,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 174,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.09546917920176193,
+      "full_absorption_rate": 0.007821901323706379,
+      "num_full_absorption": 13,
+      "num_probe_true_positives": 1662,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.44667699863373395,
+      "full_absorption_rate": 0.10953562566465792,
+      "num_full_absorption": 309,
+      "num_probe_true_positives": 2821,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.09178269157296917,
+      "full_absorption_rate": 0.0011926058437686344,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 1677,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.08318646522525573,
+      "full_absorption_rate": 0.0013280212483399733,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 753,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.01805468943015878,
+      "full_absorption_rate": 0.0011876484560570072,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 842,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.046187567611688574,
+      "full_absorption_rate": 0.007680491551459293,
+      "num_full_absorption": 5,
+      "num_probe_true_positives": 651,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.1272381587024434,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 107,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.030348097021386293,
+      "full_absorption_rate": 0.010362694300518135,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 193,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.014256022636069204,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 239,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_1",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/absorption/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_2_custom_sae_eval_results.json

@@ -0,0 +1,268 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50
+  },
+  "eval_id": "8ebb674d-9f15-4d5c-8f47-48448dd21b35",
+  "datetime_epoch_millis": 1738793140191,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.19926936403983087,
+      "mean_full_absorption_score": 0.05967727904676934,
+      "mean_num_split_features": 1.0769230769230769,
+      "std_dev_absorption_fraction_score": 0.200193825605343,
+      "std_dev_full_absorption_score": 0.10500077608288533,
+      "std_dev_num_split_features": 0.271746488194703
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.37783256022605965,
+      "full_absorption_rate": 0.024437548487199378,
+      "num_full_absorption": 63,
+      "num_probe_true_positives": 2578,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.15587574948959265,
+      "full_absorption_rate": 0.007211538461538462,
+      "num_full_absorption": 12,
+      "num_probe_true_positives": 1664,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.5719603969783236,
+      "full_absorption_rate": 0.2580757207363668,
+      "num_full_absorption": 743,
+      "num_probe_true_positives": 2879,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.5621089686749688,
+      "full_absorption_rate": 0.2672209026128266,
+      "num_full_absorption": 450,
+      "num_probe_true_positives": 1684,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.16834370820991082,
+      "full_absorption_rate": 0.05699481865284974,
+      "num_full_absorption": 88,
+      "num_probe_true_positives": 1544,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.11667156140480404,
+      "full_absorption_rate": 0.018056749785038694,
+      "num_full_absorption": 21,
+      "num_probe_true_positives": 1163,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.06481540727629757,
+      "full_absorption_rate": 0.0054446460980036296,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 1102,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.06261331028463099,
+      "full_absorption_rate": 0.0009910802775024777,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 1009,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.24185250920050555,
+      "full_absorption_rate": 0.039298669891172915,
+      "num_full_absorption": 65,
+      "num_probe_true_positives": 1654,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.016041299343090145,
+      "full_absorption_rate": 0.0024630541871921183,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 406,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.012157821213304845,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 665,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.13470793723735,
+      "full_absorption_rate": 0.015899581589958158,
+      "num_full_absorption": 19,
+      "num_probe_true_positives": 1195,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.20217104889211931,
+      "full_absorption_rate": 0.017089305402425578,
+      "num_full_absorption": 31,
+      "num_probe_true_positives": 1814,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.06165938260438851,
+      "full_absorption_rate": 0.007556675062972292,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 794,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.17666919527397562,
+      "full_absorption_rate": 0.05142857142857143,
+      "num_full_absorption": 54,
+      "num_probe_true_positives": 1050,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.602060568057986,
+      "full_absorption_rate": 0.20108695652173914,
+      "num_full_absorption": 481,
+      "num_probe_true_positives": 2392,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.00665463288277255,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 174,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.2618176100631538,
+      "full_absorption_rate": 0.05174488567990373,
+      "num_full_absorption": 86,
+      "num_probe_true_positives": 1662,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.6729389684863054,
+      "full_absorption_rate": 0.4143920595533499,
+      "num_full_absorption": 1169,
+      "num_probe_true_positives": 2821,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.212218166643232,
+      "full_absorption_rate": 0.015503875968992248,
+      "num_full_absorption": 26,
+      "num_probe_true_positives": 1677,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.2805675145009456,
+      "full_absorption_rate": 0.07702523240371846,
+      "num_full_absorption": 58,
+      "num_probe_true_positives": 753,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.02793276716008858,
+      "full_absorption_rate": 0.004750593824228029,
+      "num_full_absorption": 4,
+      "num_probe_true_positives": 842,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.0646989645046496,
+      "full_absorption_rate": 0.010752688172043012,
+      "num_full_absorption": 7,
+      "num_probe_true_positives": 651,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.08669041060559739,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 107,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.030923398913955746,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 193,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.009019606907593795,
+      "full_absorption_rate": 0.0041841004184100415,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 239,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_2",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/absorption/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_3_custom_sae_eval_results.json

@@ -0,0 +1,268 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50
+  },
+  "eval_id": "8ac82cb6-4d80-4788-b030-d3ff9ecabc15",
+  "datetime_epoch_millis": 1738790694222,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.27380462363873964,
+      "mean_full_absorption_score": 0.11174761587082688,
+      "mean_num_split_features": 1.3846153846153846,
+      "std_dev_absorption_fraction_score": 0.2085371119401205,
+      "std_dev_full_absorption_score": 0.10395305649996002,
+      "std_dev_num_split_features": 0.8978607053178383
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.6334103628494663,
+      "full_absorption_rate": 0.28316524437548485,
+      "num_full_absorption": 730,
+      "num_probe_true_positives": 2578,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.5612972872403076,
+      "full_absorption_rate": 0.19831730769230768,
+      "num_full_absorption": 330,
+      "num_probe_true_positives": 1664,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.4456334749181992,
+      "full_absorption_rate": 0.13928447377561654,
+      "num_full_absorption": 401,
+      "num_probe_true_positives": 2879,
+      "num_split_features": 5
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.4663763563664617,
+      "full_absorption_rate": 0.1680522565320665,
+      "num_full_absorption": 283,
+      "num_probe_true_positives": 1684,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.28617098631064275,
+      "full_absorption_rate": 0.1794041450777202,
+      "num_full_absorption": 277,
+      "num_probe_true_positives": 1544,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.31944391642843717,
+      "full_absorption_rate": 0.13929492691315562,
+      "num_full_absorption": 162,
+      "num_probe_true_positives": 1163,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.326763907694398,
+      "full_absorption_rate": 0.1705989110707804,
+      "num_full_absorption": 188,
+      "num_probe_true_positives": 1102,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.11808222242991506,
+      "full_absorption_rate": 0.018830525272547076,
+      "num_full_absorption": 19,
+      "num_probe_true_positives": 1009,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.33518338041432005,
+      "full_absorption_rate": 0.1505441354292624,
+      "num_full_absorption": 249,
+      "num_probe_true_positives": 1654,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.03867125135736307,
+      "full_absorption_rate": 0.0024630541871921183,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 406,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.02708629091329992,
+      "full_absorption_rate": 0.009022556390977444,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 665,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.27172567963607736,
+      "full_absorption_rate": 0.11799163179916318,
+      "num_full_absorption": 141,
+      "num_probe_true_positives": 1195,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.500443622143782,
+      "full_absorption_rate": 0.19570011025358325,
+      "num_full_absorption": 355,
+      "num_probe_true_positives": 1814,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.13894620345252578,
+      "full_absorption_rate": 0.03904282115869018,
+      "num_full_absorption": 31,
+      "num_probe_true_positives": 794,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.20516155603410746,
+      "full_absorption_rate": 0.11619047619047619,
+      "num_full_absorption": 122,
+      "num_probe_true_positives": 1050,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.6877451457818394,
+      "full_absorption_rate": 0.3520066889632107,
+      "num_full_absorption": 842,
+      "num_probe_true_positives": 2392,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.017662373425384282,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 174,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.31924702002139765,
+      "full_absorption_rate": 0.07220216606498195,
+      "num_full_absorption": 120,
+      "num_probe_true_positives": 1662,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.5966041750062926,
+      "full_absorption_rate": 0.3342786246012052,
+      "num_full_absorption": 943,
+      "num_probe_true_positives": 2821,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.33112850242087766,
+      "full_absorption_rate": 0.11150864639236732,
+      "num_full_absorption": 187,
+      "num_probe_true_positives": 1677,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.1736379175397297,
+      "full_absorption_rate": 0.0451527224435591,
+      "num_full_absorption": 34,
+      "num_probe_true_positives": 753,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.06615323067748713,
+      "full_absorption_rate": 0.020190023752969122,
+      "num_full_absorption": 17,
+      "num_probe_true_positives": 842,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.11299054531700574,
+      "full_absorption_rate": 0.027649769585253458,
+      "num_full_absorption": 18,
+      "num_probe_true_positives": 651,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.09675091301510899,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 107,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.04066091772139128,
+      "full_absorption_rate": 0.010362694300518135,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 193,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.0019429754914134399,
+      "full_absorption_rate": 0.0041841004184100415,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 239,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_3",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/absorption/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_4_custom_sae_eval_results.json

@@ -0,0 +1,268 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50
+  },
+  "eval_id": "a6c157c2-8266-4539-a364-d4f3d4b95e63",
+  "datetime_epoch_millis": 1738795756067,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.2970629494295943,
+      "mean_full_absorption_score": 0.16060743819078988,
+      "mean_num_split_features": 1.6923076923076923,
+      "std_dev_absorption_fraction_score": 0.21833356962674993,
+      "std_dev_full_absorption_score": 0.13683720697531063,
+      "std_dev_num_split_features": 1.2575923272422036
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.4413904461749635,
+      "full_absorption_rate": 0.16446858029480219,
+      "num_full_absorption": 424,
+      "num_probe_true_positives": 2578,
+      "num_split_features": 5
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.450685431712029,
+      "full_absorption_rate": 0.18088942307692307,
+      "num_full_absorption": 301,
+      "num_probe_true_positives": 1664,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.5821904564134256,
+      "full_absorption_rate": 0.30670371656825285,
+      "num_full_absorption": 883,
+      "num_probe_true_positives": 2879,
+      "num_split_features": 5
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.5742337169635329,
+      "full_absorption_rate": 0.32185273159144895,
+      "num_full_absorption": 542,
+      "num_probe_true_positives": 1684,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.18981298174844446,
+      "full_absorption_rate": 0.13471502590673576,
+      "num_full_absorption": 208,
+      "num_probe_true_positives": 1544,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.4303705672075229,
+      "full_absorption_rate": 0.24333619948409285,
+      "num_full_absorption": 283,
+      "num_probe_true_positives": 1163,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.39300790034337457,
+      "full_absorption_rate": 0.26406533575317603,
+      "num_full_absorption": 291,
+      "num_probe_true_positives": 1102,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.1473633705426197,
+      "full_absorption_rate": 0.04162537165510406,
+      "num_full_absorption": 42,
+      "num_probe_true_positives": 1009,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.2615839241768494,
+      "full_absorption_rate": 0.14691656590084642,
+      "num_full_absorption": 243,
+      "num_probe_true_positives": 1654,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.05117406667896947,
+      "full_absorption_rate": 0.007389162561576354,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 406,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.03448794297537904,
+      "full_absorption_rate": 0.01804511278195489,
+      "num_full_absorption": 12,
+      "num_probe_true_positives": 665,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.528743346032827,
+      "full_absorption_rate": 0.37489539748953976,
+      "num_full_absorption": 448,
+      "num_probe_true_positives": 1195,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.38150848462448217,
+      "full_absorption_rate": 0.1703417861080485,
+      "num_full_absorption": 309,
+      "num_probe_true_positives": 1814,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.21705599444161572,
+      "full_absorption_rate": 0.0818639798488665,
+      "num_full_absorption": 65,
+      "num_probe_true_positives": 794,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.19684352710791495,
+      "full_absorption_rate": 0.14476190476190476,
+      "num_full_absorption": 152,
+      "num_probe_true_positives": 1050,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.756351767767991,
+      "full_absorption_rate": 0.4744983277591973,
+      "num_full_absorption": 1135,
+      "num_probe_true_positives": 2392,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.022738719106524842,
+      "full_absorption_rate": 0.005747126436781609,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 174,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.5716800098523727,
+      "full_absorption_rate": 0.32912154031287605,
+      "num_full_absorption": 547,
+      "num_probe_true_positives": 1662,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.5491532507334174,
+      "full_absorption_rate": 0.3548387096774194,
+      "num_full_absorption": 1001,
+      "num_probe_true_positives": 2821,
+      "num_split_features": 4
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.4160762059426235,
+      "full_absorption_rate": 0.2081097197376267,
+      "num_full_absorption": 349,
+      "num_probe_true_positives": 1677,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.17104284092706945,
+      "full_absorption_rate": 0.07171314741035857,
+      "num_full_absorption": 54,
+      "num_probe_true_positives": 753,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.08154865944312527,
+      "full_absorption_rate": 0.032066508313539195,
+      "num_full_absorption": 27,
+      "num_probe_true_positives": 842,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.14250459566400003,
+      "full_absorption_rate": 0.05837173579109063,
+      "num_full_absorption": 38,
+      "num_probe_true_positives": 651,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.06885068971289977,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 107,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.061906340273470535,
+      "full_absorption_rate": 0.031088082901554404,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 193,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.0013314486020077486,
+      "full_absorption_rate": 0.008368200836820083,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 239,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_4",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/absorption/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_5_custom_sae_eval_results.json

@@ -0,0 +1,268 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50
+  },
+  "eval_id": "59329e73-810f-472d-8ab5-7d0b59ee8032",
+  "datetime_epoch_millis": 1738788312573,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.27616667840053044,
+      "mean_full_absorption_score": 0.18311710219845018,
+      "mean_num_split_features": 2.0,
+      "std_dev_absorption_fraction_score": 0.17955401555761985,
+      "std_dev_full_absorption_score": 0.12841765476462225,
+      "std_dev_num_split_features": 1.6
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.4135742482669448,
+      "full_absorption_rate": 0.21644685802948022,
+      "num_full_absorption": 558,
+      "num_probe_true_positives": 2578,
+      "num_split_features": 4
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.4015526296583575,
+      "full_absorption_rate": 0.20072115384615385,
+      "num_full_absorption": 334,
+      "num_probe_true_positives": 1664,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.4404130651991934,
+      "full_absorption_rate": 0.2549496352900313,
+      "num_full_absorption": 734,
+      "num_probe_true_positives": 2879,
+      "num_split_features": 6
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.4588083161124332,
+      "full_absorption_rate": 0.28028503562945367,
+      "num_full_absorption": 472,
+      "num_probe_true_positives": 1684,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.1447774581665048,
+      "full_absorption_rate": 0.13147668393782383,
+      "num_full_absorption": 203,
+      "num_probe_true_positives": 1544,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.3873865408321871,
+      "full_absorption_rate": 0.2656921754084265,
+      "num_full_absorption": 309,
+      "num_probe_true_positives": 1163,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.2375981200227858,
+      "full_absorption_rate": 0.16696914700544466,
+      "num_full_absorption": 184,
+      "num_probe_true_positives": 1102,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.18660967928208133,
+      "full_absorption_rate": 0.0802775024777007,
+      "num_full_absorption": 81,
+      "num_probe_true_positives": 1009,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.3631443822738456,
+      "full_absorption_rate": 0.29081015719467956,
+      "num_full_absorption": 481,
+      "num_probe_true_positives": 1654,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.07115954525673361,
+      "full_absorption_rate": 0.017241379310344827,
+      "num_full_absorption": 7,
+      "num_probe_true_positives": 406,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.04879204937766494,
+      "full_absorption_rate": 0.031578947368421054,
+      "num_full_absorption": 21,
+      "num_probe_true_positives": 665,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.4822787469830486,
+      "full_absorption_rate": 0.401673640167364,
+      "num_full_absorption": 480,
+      "num_probe_true_positives": 1195,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.38377290557583654,
+      "full_absorption_rate": 0.2552370452039691,
+      "num_full_absorption": 463,
+      "num_probe_true_positives": 1814,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.17978290229285845,
+      "full_absorption_rate": 0.10327455919395466,
+      "num_full_absorption": 82,
+      "num_probe_true_positives": 794,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.19812942766943445,
+      "full_absorption_rate": 0.17047619047619048,
+      "num_full_absorption": 179,
+      "num_probe_true_positives": 1050,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.6056222996145454,
+      "full_absorption_rate": 0.3448996655518395,
+      "num_full_absorption": 825,
+      "num_probe_true_positives": 2392,
+      "num_split_features": 5
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.03644851552466968,
+      "full_absorption_rate": 0.022988505747126436,
+      "num_full_absorption": 4,
+      "num_probe_true_positives": 174,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.5460252773593768,
+      "full_absorption_rate": 0.3802647412755716,
+      "num_full_absorption": 632,
+      "num_probe_true_positives": 1662,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.43503717196696245,
+      "full_absorption_rate": 0.2977667493796526,
+      "num_full_absorption": 840,
+      "num_probe_true_positives": 2821,
+      "num_split_features": 6
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.37168244953794066,
+      "full_absorption_rate": 0.24388789505068575,
+      "num_full_absorption": 409,
+      "num_probe_true_positives": 1677,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.4015811028782945,
+      "full_absorption_rate": 0.3930942895086321,
+      "num_full_absorption": 296,
+      "num_probe_true_positives": 753,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.09352294964084032,
+      "full_absorption_rate": 0.060570071258907364,
+      "num_full_absorption": 51,
+      "num_probe_true_positives": 842,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.16147162387821953,
+      "full_absorption_rate": 0.08294930875576037,
+      "num_full_absorption": 54,
+      "num_probe_true_positives": 651,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.08565750731693882,
+      "full_absorption_rate": 0.018691588785046728,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 107,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.04526030996735048,
+      "full_absorption_rate": 0.03626943005181347,
+      "num_full_absorption": 7,
+      "num_probe_true_positives": 193,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.00024441375874335945,
+      "full_absorption_rate": 0.012552301255230125,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 239,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_5",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/autointerp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_0_custom_sae_eval_results.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fade7ed161c5db9c7c7ac163c599bd3859bb24ecc70c0a1c824a46d9e37fda6d
+size 27824414

old_relu_eval_results/autointerp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_1_custom_sae_eval_results.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5adce21b197509292cdd2f4da8895a5762bf93e1f0b2fbcd3ba2daf6d93f8e9d
+size 27583837

old_relu_eval_results/autointerp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_2_custom_sae_eval_results.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2e3c88b391388eccb6fa9d88e5f912d3862401c3acef072ed8a20527e0ac372
+size 27435249

old_relu_eval_results/autointerp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_3_custom_sae_eval_results.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f9d90013affb818d37b90201097e3c443522b80e5e3832fe9715633decb95ec
+size 27194850

old_relu_eval_results/autointerp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_4_custom_sae_eval_results.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8acd6328873671a729f89408de8ca580ec5b368493dabd9363953487b74019b8
+size 27102881

old_relu_eval_results/autointerp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_5_custom_sae_eval_results.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a72d03c8276dce0adb0b65cdecd992edbff8cc4f97a61d17ad63b7e936c53cb
+size 26823026

old_relu_eval_results/scr/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_0_custom_sae_eval_results.json

@@ -0,0 +1,323 @@
+{
+  "eval_type_id": "scr",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": true,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "f84c237b-b63d-44d6-9184-83af39eb778e",
+  "datetime_epoch_millis": 1738800834845,
+  "eval_result_metrics": {
+    "scr_metrics": {
+      "scr_dir1_threshold_2": 0.23234423354969463,
+      "scr_metric_threshold_2": 0.09820290124961623,
+      "scr_dir2_threshold_2": 0.10051744442943668,
+      "scr_dir1_threshold_5": 0.22836236745436214,
+      "scr_metric_threshold_5": 0.1612618897051168,
+      "scr_dir2_threshold_5": 0.16833831095315674,
+      "scr_dir1_threshold_10": 0.23399130656918807,
+      "scr_metric_threshold_10": 0.21393385636206694,
+      "scr_dir2_threshold_10": 0.2210026206525063,
+      "scr_dir1_threshold_20": 0.24236966949927263,
+      "scr_metric_threshold_20": 0.26432320579946816,
+      "scr_dir2_threshold_20": 0.26846685069449183,
+      "scr_dir1_threshold_50": 0.21470674119765001,
+      "scr_metric_threshold_50": 0.3265154510614142,
+      "scr_dir2_threshold_50": 0.3172547508429693,
+      "scr_dir1_threshold_100": 0.20032932591386368,
+      "scr_metric_threshold_100": 0.38983340792408966,
+      "scr_dir2_threshold_100": 0.38098404313994894,
+      "scr_dir1_threshold_500": 0.037709607495273495,
+      "scr_metric_threshold_500": 0.3428859758427782,
+      "scr_dir2_threshold_500": 0.34688913659267395
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_professor_nurse_results",
+      "scr_dir1_threshold_2": 0.4687504074535596,
+      "scr_metric_threshold_2": 0.012315258641572036,
+      "scr_dir2_threshold_2": 0.012315258641572036,
+      "scr_dir1_threshold_5": 0.5156247962732202,
+      "scr_metric_threshold_5": 0.022167494916722333,
+      "scr_dir2_threshold_5": 0.022167494916722333,
+      "scr_dir1_threshold_10": 0.578124912688523,
+      "scr_metric_threshold_10": 0.03201958438240929,
+      "scr_dir2_threshold_10": 0.03201958438240929,
+      "scr_dir1_threshold_20": 0.4843752037267798,
+      "scr_metric_threshold_20": 0.06403931557428191,
+      "scr_dir2_threshold_20": 0.06403931557428191,
+      "scr_dir1_threshold_50": 0.5,
+      "scr_metric_threshold_50": 0.08128076575816078,
+      "scr_dir2_threshold_50": 0.08128076575816078,
+      "scr_dir1_threshold_100": 0.43749988358469727,
+      "scr_metric_threshold_100": 0.10098509149899802,
+      "scr_dir2_threshold_100": 0.10098509149899802,
+      "scr_dir1_threshold_500": 0.3593749708961743,
+      "scr_metric_threshold_500": 0.16748757624916502,
+      "scr_dir2_threshold_500": 0.16748757624916502
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
+      "scr_dir1_threshold_2": 0.4205609298770807,
+      "scr_metric_threshold_2": 0.1239194296368033,
+      "scr_dir2_threshold_2": 0.1239194296368033,
+      "scr_dir1_threshold_5": 0.35514009320607276,
+      "scr_metric_threshold_5": 0.1556196529738595,
+      "scr_dir2_threshold_5": 0.1556196529738595,
+      "scr_dir1_threshold_10": 0.30841116289226267,
+      "scr_metric_threshold_10": 0.23342947946078924,
+      "scr_dir2_threshold_10": 0.23342947946078924,
+      "scr_dir1_threshold_20": 0.2616822325784525,
+      "scr_metric_threshold_20": 0.2881845043727822,
+      "scr_dir2_threshold_20": 0.2881845043727822,
+      "scr_dir1_threshold_50": 0.17757004660303638,
+      "scr_metric_threshold_50": 0.37463975262183136,
+      "scr_dir2_threshold_50": 0.37463975262183136,
+      "scr_dir1_threshold_100": 0.08411218597541614,
+      "scr_metric_threshold_100": 0.4783861879377377,
+      "scr_dir2_threshold_100": 0.4783861879377377,
+      "scr_dir1_threshold_500": -0.6168223257845253,
+      "scr_metric_threshold_500": 0.18155626180283607,
+      "scr_dir2_threshold_500": 0.18155626180283607
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
+      "scr_dir1_threshold_2": 0.5156247962732202,
+      "scr_metric_threshold_2": 0.03544318161412681,
+      "scr_dir2_threshold_2": 0.03544318161412681,
+      "scr_dir1_threshold_5": 0.4843752037267798,
+      "scr_metric_threshold_5": 0.05063300880745302,
+      "scr_dir2_threshold_5": 0.05063300880745302,
+      "scr_dir1_threshold_10": 0.4843752037267798,
+      "scr_metric_threshold_10": 0.09620264128525105,
+      "scr_dir2_threshold_10": 0.09620264128525105,
+      "scr_dir1_threshold_20": 0.5156247962732202,
+      "scr_metric_threshold_20": 0.12911405928564068,
+      "scr_dir2_threshold_20": 0.12911405928564068,
+      "scr_dir1_threshold_50": 0.39062549476503666,
+      "scr_metric_threshold_50": 0.1797469171952743,
+      "scr_dir2_threshold_50": 0.1797469171952743,
+      "scr_dir1_threshold_100": 0.32812537834973393,
+      "scr_metric_threshold_100": 0.23797476415048074,
+      "scr_dir2_threshold_100": 0.23797476415048074,
+      "scr_dir1_threshold_500": -0.12499930150818353,
+      "scr_metric_threshold_500": 0.07848105047818764,
+      "scr_dir2_threshold_500": 0.07848105047818764
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
+      "scr_dir1_threshold_2": 0.34645683334217237,
+      "scr_metric_threshold_2": 0.06528194057151832,
+      "scr_dir2_threshold_2": 0.06528194057151832,
+      "scr_dir1_threshold_5": 0.26771677194471133,
+      "scr_metric_threshold_5": 0.13946602022454502,
+      "scr_dir2_threshold_5": 0.13946602022454502,
+      "scr_dir1_threshold_10": 0.21259844736981667,
+      "scr_metric_threshold_10": 0.1869436826330466,
+      "scr_dir2_threshold_10": 0.1869436826330466,
+      "scr_dir1_threshold_20": 0.18110232894527498,
+      "scr_metric_threshold_20": 0.2670624037614951,
+      "scr_dir2_threshold_20": 0.2670624037614951,
+      "scr_dir1_threshold_50": -0.07086614912327223,
+      "scr_metric_threshold_50": 0.3412463065461462,
+      "scr_dir2_threshold_50": 0.3412463065461462,
+      "scr_dir1_threshold_100": -0.14960621052073325,
+      "scr_metric_threshold_100": 0.42433234841230566,
+      "scr_dir2_threshold_100": 0.42433234841230566,
+      "scr_dir1_threshold_500": -0.4724408377125527,
+      "scr_metric_threshold_500": 0.08011872112844853,
+      "scr_dir2_threshold_500": 0.08011872112844853
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
+      "scr_dir1_threshold_2": 0.021739172687599125,
+      "scr_metric_threshold_2": 0.17647042323973455,
+      "scr_dir2_threshold_2": 0.17647042323973455,
+      "scr_dir1_threshold_5": 0.010869424374669583,
+      "scr_metric_threshold_5": 0.38039217519558505,
+      "scr_dir2_threshold_5": 0.38039217519558505,
+      "scr_dir1_threshold_10": -0.021739172687599125,
+      "scr_metric_threshold_10": 0.5058823941900663,
+      "scr_dir2_threshold_10": 0.5058823941900663,
+      "scr_dir1_threshold_20": -0.005434712187334791,
+      "scr_metric_threshold_20": 0.5647058686033112,
+      "scr_dir2_threshold_20": 0.5647058686033112,
+      "scr_dir1_threshold_50": -0.07608694243746696,
+      "scr_metric_threshold_50": 0.6470586860331121,
+      "scr_dir2_threshold_50": 0.6470586860331121,
+      "scr_dir1_threshold_100": -0.059782805875462586,
+      "scr_metric_threshold_100": 0.6941176058099336,
+      "scr_dir2_threshold_100": 0.6941176058099336,
+      "scr_dir1_threshold_500": 0.010869424374669583,
+      "scr_metric_threshold_500": 0.7411765255867552,
+      "scr_dir2_threshold_500": 0.7411765255867552
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
+      "scr_dir1_threshold_2": 0.015463717977866399,
+      "scr_metric_threshold_2": 0.08467752208135552,
+      "scr_dir2_threshold_2": 0.08467752208135552,
+      "scr_dir1_threshold_5": 0.08247408426990457,
+      "scr_metric_threshold_5": 0.12096785435318151,
+      "scr_dir2_threshold_5": 0.12096785435318151,
+      "scr_dir1_threshold_10": 0.1340204253436484,
+      "scr_metric_threshold_10": 0.15322574055158325,
+      "scr_dir2_threshold_10": 0.15322574055158325,
+      "scr_dir1_threshold_20": 0.2061853642949754,
+      "scr_metric_threshold_20": 0.20161293036111277,
+      "scr_dir2_threshold_20": 0.20161293036111277,
+      "scr_dir1_threshold_50": 0.24742256005014163,
+      "scr_metric_threshold_50": 0.2661291834404855,
+      "scr_dir2_threshold_50": 0.2661291834404855,
+      "scr_dir1_threshold_100": 0.2886597558053079,
+      "scr_metric_threshold_100": 0.3508064651805564,
+      "scr_dir2_threshold_100": 0.3508064651805564,
+      "scr_dir1_threshold_500": 0.2886597558053079,
+      "scr_metric_threshold_500": 0.5564516012738088,
+      "scr_dir2_threshold_500": 0.5564516012738088
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
+      "scr_dir1_threshold_2": 0.03153139365860286,
+      "scr_metric_threshold_2": 0.2488888370843636,
+      "scr_dir2_threshold_2": 0.2488888370843636,
+      "scr_dir1_threshold_5": 0.07207194871208326,
+      "scr_metric_threshold_5": 0.38222229404213226,
+      "scr_dir2_threshold_5": 0.38222229404213226,
+      "scr_dir1_threshold_10": 0.09459444946519599,
+      "scr_metric_threshold_10": 0.42222230463851246,
+      "scr_dir2_threshold_10": 0.42222230463851246,
+      "scr_dir1_threshold_20": 0.16666666666666666,
+      "scr_metric_threshold_20": 0.4711110887409752,
+      "scr_dir2_threshold_20": 0.4711110887409752,
+      "scr_dir1_threshold_50": 0.35585583408644605,
+      "scr_metric_threshold_50": 0.5288889112590248,
+      "scr_dir2_threshold_50": 0.5288889112590248,
+      "scr_dir1_threshold_100": 0.45045055204102946,
+      "scr_metric_threshold_100": 0.6088889324517851,
+      "scr_dir2_threshold_100": 0.6088889324517851,
+      "scr_dir1_threshold_500": 0.6081080573128186,
+      "scr_metric_threshold_500": 0.6888889536445455,
+      "scr_dir2_threshold_500": 0.6888889536445455
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
+      "scr_dir1_threshold_2": 0.03862661712745569,
+      "scr_metric_threshold_2": 0.03862661712745569,
+      "scr_dir2_threshold_2": 0.05714296256601923,
+      "scr_dir1_threshold_5": 0.03862661712745569,
+      "scr_metric_threshold_5": 0.03862661712745569,
+      "scr_dir2_threshold_5": 0.09523798711177515,
+      "scr_dir1_threshold_10": 0.08154502375487709,
+      "scr_metric_threshold_10": 0.08154502375487709,
+      "scr_dir2_threshold_10": 0.13809513807839202,
+      "scr_dir1_threshold_20": 0.12875547569614632,
+      "scr_metric_threshold_20": 0.12875547569614632,
+      "scr_dir2_threshold_20": 0.1619046348563358,
+      "scr_dir1_threshold_50": 0.19313308563727843,
+      "scr_metric_threshold_50": 0.19313308563727843,
+      "scr_dir2_threshold_50": 0.11904748388971893,
+      "scr_dir1_threshold_100": 0.22317586795092056,
+      "scr_metric_threshold_100": 0.22317586795092056,
+      "scr_dir2_threshold_100": 0.1523809496777944,
+      "scr_dir1_threshold_500": 0.2489271165784791,
+      "scr_metric_threshold_500": 0.2489271165784791,
+      "scr_dir2_threshold_500": 0.280952402577645
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_0",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/scr/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_1_custom_sae_eval_results.json

@@ -0,0 +1,323 @@
+{
+  "eval_type_id": "scr",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": true,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "e55f5870-c524-4a98-bbbd-3745c4e6ba5f",
+  "datetime_epoch_millis": 1738801279714,
+  "eval_result_metrics": {
+    "scr_metrics": {
+      "scr_dir1_threshold_2": 0.23710567889490614,
+      "scr_metric_threshold_2": 0.09984043514036153,
+      "scr_dir2_threshold_2": 0.10096448219391552,
+      "scr_dir1_threshold_5": 0.23277434004406478,
+      "scr_metric_threshold_5": 0.1533444054899716,
+      "scr_dir2_threshold_5": 0.1564892090403169,
+      "scr_dir1_threshold_10": 0.25301255397775474,
+      "scr_metric_threshold_10": 0.21059082421971573,
+      "scr_dir2_threshold_10": 0.2181373055614877,
+      "scr_dir1_threshold_20": 0.24499730767158928,
+      "scr_metric_threshold_20": 0.2566161420240508,
+      "scr_dir2_threshold_20": 0.2605247568722085,
+      "scr_dir1_threshold_50": 0.22983255987764495,
+      "scr_metric_threshold_50": 0.32999437667348414,
+      "scr_dir2_threshold_50": 0.3275827600927625,
+      "scr_dir1_threshold_100": 0.1800306346905109,
+      "scr_metric_threshold_100": 0.38054020119475096,
+      "scr_dir2_threshold_100": 0.37907635825688035,
+      "scr_dir1_threshold_500": 0.0590895362527501,
+      "scr_metric_threshold_500": 0.3194485925466525,
+      "scr_dir2_threshold_500": 0.33138143776464996
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_professor_nurse_results",
+      "scr_dir1_threshold_2": 0.4531256111803394,
+      "scr_metric_threshold_2": 0.017241303374415515,
+      "scr_dir2_threshold_2": 0.017241303374415515,
+      "scr_dir1_threshold_5": 0.4687504074535596,
+      "scr_metric_threshold_5": 0.02709353964956581,
+      "scr_dir2_threshold_5": 0.02709353964956581,
+      "scr_dir1_threshold_10": 0.5,
+      "scr_metric_threshold_10": 0.039408798291137845,
+      "scr_dir2_threshold_10": 0.039408798291137845,
+      "scr_dir1_threshold_20": 0.5312505238688624,
+      "scr_metric_threshold_20": 0.0467980121998664,
+      "scr_dir2_threshold_20": 0.0467980121998664,
+      "scr_dir1_threshold_50": 0.4843752037267798,
+      "scr_metric_threshold_50": 0.08374378812458251,
+      "scr_dir2_threshold_50": 0.08374378812458251,
+      "scr_dir1_threshold_100": 0.3125005820765137,
+      "scr_metric_threshold_100": 0.09113300203331107,
+      "scr_dir2_threshold_100": 0.09113300203331107,
+      "scr_dir1_threshold_500": 0.15624982537704588,
+      "scr_metric_threshold_500": 0.16009850914989981,
+      "scr_dir2_threshold_500": 0.16009850914989981
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
+      "scr_dir1_threshold_2": 0.411214698172087,
+      "scr_metric_threshold_2": 0.1268012368908431,
+      "scr_dir2_threshold_2": 0.1268012368908431,
+      "scr_dir1_threshold_5": 0.3271025121966708,
+      "scr_metric_threshold_5": 0.19308366259030446,
+      "scr_dir2_threshold_5": 0.19308366259030446,
+      "scr_dir1_threshold_10": 0.25233655792624843,
+      "scr_metric_threshold_10": 0.2881845043727822,
+      "scr_dir2_threshold_10": 0.2881845043727822,
+      "scr_dir1_threshold_20": 0.1588786972986282,
+      "scr_metric_threshold_20": 0.311239305947719,
+      "scr_dir2_threshold_20": 0.311239305947719,
+      "scr_dir1_threshold_50": 0.19626139590744457,
+      "scr_metric_threshold_50": 0.4034583404761569,
+      "scr_dir2_threshold_50": 0.4034583404761569,
+      "scr_dir1_threshold_100": 0.056074604966014206,
+      "scr_metric_threshold_100": 0.4755043806836979,
+      "scr_dir2_threshold_100": 0.4755043806836979,
+      "scr_dir1_threshold_500": 0.09345786062762024,
+      "scr_metric_threshold_500": 0.1556196529738595,
+      "scr_dir2_threshold_500": 0.1556196529738595
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
+      "scr_dir1_threshold_2": 0.5312505238688624,
+      "scr_metric_threshold_2": 0.030379805284471813,
+      "scr_dir2_threshold_2": 0.030379805284471813,
+      "scr_dir1_threshold_5": 0.5156247962732202,
+      "scr_metric_threshold_5": 0.058227846955206435,
+      "scr_dir2_threshold_5": 0.058227846955206435,
+      "scr_dir1_threshold_10": 0.5156247962732202,
+      "scr_metric_threshold_10": 0.09873425400116885,
+      "scr_dir2_threshold_10": 0.09873425400116885,
+      "scr_dir1_threshold_20": 0.4843752037267798,
+      "scr_metric_threshold_20": 0.12911405928564068,
+      "scr_dir2_threshold_20": 0.12911405928564068,
+      "scr_dir1_threshold_50": 0.39062549476503666,
+      "scr_metric_threshold_50": 0.1848101426271099,
+      "scr_dir2_threshold_50": 0.1848101426271099,
+      "scr_dir1_threshold_100": 0.17187555297268803,
+      "scr_metric_threshold_100": 0.23797476415048074,
+      "scr_dir2_threshold_100": 0.23797476415048074,
+      "scr_dir1_threshold_500": -0.5468743888196606,
+      "scr_metric_threshold_500": 0.09873425400116885,
+      "scr_dir2_threshold_500": 0.09873425400116885
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
+      "scr_dir1_threshold_2": 0.33858292106798354,
+      "scr_metric_threshold_2": 0.09198818094766784,
+      "scr_dir2_threshold_2": 0.09198818094766784,
+      "scr_dir1_threshold_5": 0.259842390342736,
+      "scr_metric_threshold_5": 0.15430280078147524,
+      "scr_dir2_threshold_5": 0.15430280078147524,
+      "scr_dir1_threshold_10": 0.2834645964930889,
+      "scr_metric_threshold_10": 0.2136499230091961,
+      "scr_dir2_threshold_10": 0.2136499230091961,
+      "scr_dir1_threshold_20": 0.16535450439689736,
+      "scr_metric_threshold_20": 0.249258302466854,
+      "scr_dir2_threshold_20": 0.249258302466854,
+      "scr_dir1_threshold_50": 0.03149611842454171,
+      "scr_metric_threshold_50": 0.3531157663653655,
+      "scr_dir2_threshold_50": 0.3531157663653655,
+      "scr_dir1_threshold_100": -0.03149611842454171,
+      "scr_metric_threshold_100": 0.4154303861991729,
+      "scr_dir2_threshold_100": 0.4154303861991729,
+      "scr_dir1_threshold_500": -0.10236179822002743,
+      "scr_metric_threshold_500": 0.09198818094766784,
+      "scr_dir2_threshold_500": 0.09198818094766784
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
+      "scr_dir1_threshold_2": 0.03260859706226871,
+      "scr_metric_threshold_2": 0.18431369340772616,
+      "scr_dir2_threshold_2": 0.18431369340772616,
+      "scr_dir1_threshold_5": 0.0380433092496035,
+      "scr_metric_threshold_5": 0.25490195620110373,
+      "scr_dir2_threshold_5": 0.25490195620110373,
+      "scr_dir1_threshold_10": 0.07065223025013216,
+      "scr_metric_threshold_10": 0.4274508612286972,
+      "scr_dir2_threshold_10": 0.4274508612286972,
+      "scr_dir1_threshold_20": 0.1195652878126652,
+      "scr_metric_threshold_20": 0.5294117372066224,
+      "scr_dir2_threshold_20": 0.5294117372066224,
+      "scr_dir1_threshold_50": 0.016304136562004374,
+      "scr_metric_threshold_50": 0.623529343016556,
+      "scr_dir2_threshold_50": 0.623529343016556,
+      "scr_dir1_threshold_100": 0.010869424374669583,
+      "scr_metric_threshold_100": 0.6784312992176598,
+      "scr_dir2_threshold_100": 0.6784312992176598,
+      "scr_dir1_threshold_500": 0.11413025168707046,
+      "scr_metric_threshold_500": 0.6980391240220748,
+      "scr_dir2_threshold_500": 0.6980391240220748
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
+      "scr_dir1_threshold_2": 0.04639146117402712,
+      "scr_metric_threshold_2": 0.060483807005948444,
+      "scr_dir2_threshold_2": 0.060483807005948444,
+      "scr_dir1_threshold_5": 0.0979381094881989,
+      "scr_metric_threshold_5": 0.14112912335516434,
+      "scr_dir2_threshold_5": 0.14112912335516434,
+      "scr_dir1_threshold_10": 0.18041219375810347,
+      "scr_metric_threshold_10": 0.15322574055158325,
+      "scr_dir2_threshold_10": 0.15322574055158325,
+      "scr_dir1_threshold_20": 0.22680396217255852,
+      "scr_metric_threshold_20": 0.21774199363095595,
+      "scr_dir2_threshold_20": 0.21774199363095595,
+      "scr_dir1_threshold_50": 0.23711341473156403,
+      "scr_metric_threshold_50": 0.2862904524424683,
+      "scr_dir2_threshold_50": 0.2862904524424683,
+      "scr_dir1_threshold_100": 0.26804115792772476,
+      "scr_metric_threshold_100": 0.3306451961785736,
+      "scr_dir2_threshold_100": 0.3306451961785736,
+      "scr_dir1_threshold_500": 0.32989664432004623,
+      "scr_metric_threshold_500": 0.5040322057321397,
+      "scr_dir2_threshold_500": 0.5040322057321397
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
+      "scr_dir1_threshold_2": 0.045045001506225466,
+      "scr_metric_threshold_2": 0.2488888370843636,
+      "scr_dir2_threshold_2": 0.2488888370843636,
+      "scr_dir1_threshold_5": 0.09909916440732847,
+      "scr_metric_threshold_5": 0.3422222834457521,
+      "scr_dir2_threshold_5": 0.3422222834457521,
+      "scr_dir1_threshold_10": 0.148648612366299,
+      "scr_metric_threshold_10": 0.39111106754821484,
+      "scr_dir2_threshold_10": 0.39111106754821484,
+      "scr_dir1_threshold_20": 0.1621622202139216,
+      "scr_metric_threshold_20": 0.45777766357234706,
+      "scr_dir2_threshold_20": 0.45777766357234706,
+      "scr_dir1_threshold_50": 0.3108108325802206,
+      "scr_metric_threshold_50": 0.5333332980120661,
+      "scr_dir2_threshold_50": 0.5333332980120661,
+      "scr_dir1_threshold_100": 0.4549549984937745,
+      "scr_metric_threshold_100": 0.6177777059578677,
+      "scr_dir2_threshold_100": 0.6177777059578677,
+      "scr_dir1_threshold_500": 0.25225222322637253,
+      "scr_metric_threshold_500": 0.671111141722876,
+      "scr_dir2_threshold_500": 0.671111141722876
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
+      "scr_dir1_threshold_2": 0.03862661712745569,
+      "scr_metric_threshold_2": 0.03862661712745569,
+      "scr_dir2_threshold_2": 0.04761899355588758,
+      "scr_dir1_threshold_5": 0.05579403094120067,
+      "scr_metric_threshold_5": 0.05579403094120067,
+      "scr_dir2_threshold_5": 0.08095245934396302,
+      "scr_dir1_threshold_10": 0.07296144475494565,
+      "scr_metric_threshold_10": 0.07296144475494565,
+      "scr_dir2_threshold_10": 0.1333332954891213,
+      "scr_dir1_threshold_20": 0.11158806188240133,
+      "scr_metric_threshold_20": 0.11158806188240133,
+      "scr_dir2_threshold_20": 0.14285698066766273,
+      "scr_dir1_threshold_50": 0.17167388232356773,
+      "scr_metric_threshold_50": 0.17167388232356773,
+      "scr_dir2_threshold_50": 0.1523809496777944,
+      "scr_dir1_threshold_100": 0.19742487513724416,
+      "scr_metric_threshold_100": 0.19742487513724416,
+      "scr_dir2_threshold_100": 0.18571413163427958,
+      "scr_dir1_threshold_500": 0.17596567182353345,
+      "scr_metric_threshold_500": 0.17596567182353345,
+      "scr_dir2_threshold_500": 0.27142843356751334
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_1",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/scr/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_2_custom_sae_eval_results.json

@@ -0,0 +1,323 @@
+{
+  "eval_type_id": "scr",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": true,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "3c85fb5d-0428-4028-827e-87b37ec72a2d",
+  "datetime_epoch_millis": 1738802658127,
+  "eval_result_metrics": {
+    "scr_metrics": {
+      "scr_dir1_threshold_2": 0.2469231071310974,
+      "scr_metric_threshold_2": 0.10476438206495488,
+      "scr_dir2_threshold_2": 0.10439652483056149,
+      "scr_dir1_threshold_5": 0.266262331969709,
+      "scr_metric_threshold_5": 0.16042571052625748,
+      "scr_dir2_threshold_5": 0.16547375379885432,
+      "scr_dir1_threshold_10": 0.2695043161578262,
+      "scr_metric_threshold_10": 0.20632414219873058,
+      "scr_dir2_threshold_10": 0.21190100220122254,
+      "scr_dir1_threshold_20": 0.2804753590978531,
+      "scr_metric_threshold_20": 0.2545814385824276,
+      "scr_dir2_threshold_20": 0.2506855391924104,
+      "scr_dir1_threshold_50": 0.22659001552042826,
+      "scr_metric_threshold_50": 0.3220650150830612,
+      "scr_dir2_threshold_50": 0.3238788383620121,
+      "scr_dir1_threshold_100": 0.11656540795529736,
+      "scr_metric_threshold_100": 0.35549666827962545,
+      "scr_dir2_threshold_100": 0.35367262506496205,
+      "scr_dir1_threshold_500": -0.15991300862854543,
+      "scr_metric_threshold_500": 0.3376438530231709,
+      "scr_dir2_threshold_500": 0.3442118974127407
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_professor_nurse_results",
+      "scr_dir1_threshold_2": 0.42187508731147705,
+      "scr_metric_threshold_2": 0.0073890670992652185,
+      "scr_dir2_threshold_2": 0.0073890670992652185,
+      "scr_dir1_threshold_5": 0.42187508731147705,
+      "scr_metric_threshold_5": 0.02709353964956581,
+      "scr_dir2_threshold_5": 0.02709353964956581,
+      "scr_dir1_threshold_10": 0.4531256111803394,
+      "scr_metric_threshold_10": 0.03448275355829437,
+      "scr_dir2_threshold_10": 0.03448275355829437,
+      "scr_dir1_threshold_20": 0.4062502910382569,
+      "scr_metric_threshold_20": 0.03694577592471611,
+      "scr_dir2_threshold_20": 0.03694577592471611,
+      "scr_dir1_threshold_50": 0.4062502910382569,
+      "scr_metric_threshold_50": 0.07881774339173903,
+      "scr_dir2_threshold_50": 0.07881774339173903,
+      "scr_dir1_threshold_100": 0.4062502910382569,
+      "scr_metric_threshold_100": 0.10591128304130484,
+      "scr_dir2_threshold_100": 0.10591128304130484,
+      "scr_dir1_threshold_500": 0.0937506402841651,
+      "scr_metric_threshold_500": 0.17241376779147183,
+      "scr_dir2_threshold_500": 0.17241376779147183
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
+      "scr_dir1_threshold_2": 0.3738319995632706,
+      "scr_metric_threshold_2": 0.1325648513989227,
+      "scr_dir2_threshold_2": 0.1325648513989227,
+      "scr_dir1_threshold_5": 0.34579441855386867,
+      "scr_metric_threshold_5": 0.19884727709838407,
+      "scr_dir2_threshold_5": 0.19884727709838407,
+      "scr_dir1_threshold_10": 0.2242989769168465,
+      "scr_metric_threshold_10": 0.2708934890772342,
+      "scr_dir2_threshold_10": 0.2708934890772342,
+      "scr_dir1_threshold_20": 0.2149533022646424,
+      "scr_metric_threshold_20": 0.2997119051602506,
+      "scr_dir2_threshold_20": 0.2997119051602506,
+      "scr_dir1_threshold_50": 0.2616822325784525,
+      "scr_metric_threshold_50": 0.40057636145080794,
+      "scr_dir2_threshold_50": 0.40057636145080794,
+      "scr_dir1_threshold_100": -0.35514009320607276,
+      "scr_metric_threshold_100": 0.4495677718547213,
+      "scr_dir2_threshold_100": 0.4495677718547213,
+      "scr_dir1_threshold_500": -1.065420836671008,
+      "scr_metric_threshold_500": 0.24783868750229743,
+      "scr_dir2_threshold_500": 0.24783868750229743
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
+      "scr_dir1_threshold_2": 0.4062502910382569,
+      "scr_metric_threshold_2": 0.0126583653752278,
+      "scr_dir2_threshold_2": 0.0126583653752278,
+      "scr_dir1_threshold_5": 0.5312505238688624,
+      "scr_metric_threshold_5": 0.030379805284471813,
+      "scr_dir2_threshold_5": 0.030379805284471813,
+      "scr_dir1_threshold_10": 0.5,
+      "scr_metric_threshold_10": 0.04810139609153522,
+      "scr_dir2_threshold_10": 0.04810139609153522,
+      "scr_dir1_threshold_20": 0.5468753201420825,
+      "scr_metric_threshold_20": 0.11139246847857727,
+      "scr_dir2_threshold_20": 0.11139246847857727,
+      "scr_dir1_threshold_50": 0.2968748544808716,
+      "scr_metric_threshold_50": 0.1620253263882109,
+      "scr_dir2_threshold_50": 0.1620253263882109,
+      "scr_dir1_threshold_100": 0.1250002328306055,
+      "scr_metric_threshold_100": 0.2050633461500911,
+      "scr_dir2_threshold_100": 0.2050633461500911,
+      "scr_dir1_threshold_500": -0.9531246798579175,
+      "scr_metric_threshold_500": 0.14683549919488467,
+      "scr_dir2_threshold_500": 0.14683549919488467
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
+      "scr_dir1_threshold_2": 0.30708680264344185,
+      "scr_metric_threshold_2": 0.10089032002917621,
+      "scr_dir2_threshold_2": 0.10089032002917621,
+      "scr_dir1_threshold_5": 0.2362206535201696,
+      "scr_metric_threshold_5": 0.14243334096225593,
+      "scr_dir2_threshold_5": 0.14243334096225593,
+      "scr_dir1_threshold_10": 0.2362206535201696,
+      "scr_metric_threshold_10": 0.2017804631899768,
+      "scr_dir2_threshold_10": 0.2017804631899768,
+      "scr_dir1_threshold_20": 0.17322841667108618,
+      "scr_metric_threshold_20": 0.26112776228607326,
+      "scr_dir2_threshold_20": 0.26112776228607326,
+      "scr_dir1_threshold_50": -0.10236179822002743,
+      "scr_metric_threshold_50": 0.3560830871030764,
+      "scr_dir2_threshold_50": 0.3560830871030764,
+      "scr_dir1_threshold_100": -0.05511785524710812,
+      "scr_metric_threshold_100": 0.412463065461462,
+      "scr_dir2_threshold_100": 0.412463065461462,
+      "scr_dir1_threshold_500": -0.259842390342736,
+      "scr_metric_threshold_500": 0.1899110033707575,
+      "scr_dir2_threshold_500": 0.1899110033707575
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
+      "scr_dir1_threshold_2": 0.048913057562533044,
+      "scr_metric_threshold_2": 0.1647058686033112,
+      "scr_dir2_threshold_2": 0.1647058686033112,
+      "scr_dir1_threshold_5": 0.08695636681213655,
+      "scr_metric_threshold_5": 0.3490195620110374,
+      "scr_dir2_threshold_5": 0.3490195620110374,
+      "scr_dir1_threshold_10": 0.08695636681213655,
+      "scr_metric_threshold_10": 0.443137167820971,
+      "scr_dir2_threshold_10": 0.443137167820971,
+      "scr_dir1_threshold_20": 0.10326082731240087,
+      "scr_metric_threshold_20": 0.5098039124022075,
+      "scr_dir2_threshold_20": 0.5098039124022075,
+      "scr_dir1_threshold_50": -0.005434712187334791,
+      "scr_metric_threshold_50": 0.6156863065922739,
+      "scr_dir2_threshold_50": 0.6156863065922739,
+      "scr_dir1_threshold_100": -0.0869566907503965,
+      "scr_metric_threshold_100": 0.6274508612286972,
+      "scr_dir2_threshold_100": 0.6274508612286972,
+      "scr_dir1_threshold_500": 0.0,
+      "scr_metric_threshold_500": 0.6901960875977925,
+      "scr_dir2_threshold_500": 0.6901960875977925
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
+      "scr_dir1_threshold_2": 0.15463902322123155,
+      "scr_metric_threshold_2": 0.08870972781349516,
+      "scr_dir2_threshold_2": 0.08870972781349516,
+      "scr_dir1_threshold_5": 0.1958762189763978,
+      "scr_metric_threshold_5": 0.1290322658174608,
+      "scr_dir2_threshold_5": 0.1290322658174608,
+      "scr_dir1_threshold_10": 0.23711341473156403,
+      "scr_metric_threshold_10": 0.1491935348194436,
+      "scr_dir2_threshold_10": 0.1491935348194436,
+      "scr_dir1_threshold_20": 0.2886597558053079,
+      "scr_metric_threshold_20": 0.20967734182539205,
+      "scr_dir2_threshold_20": 0.20967734182539205,
+      "scr_dir1_threshold_50": 0.36082469475663487,
+      "scr_metric_threshold_50": 0.25403232590278196,
+      "scr_dir2_threshold_50": 0.25403232590278196,
+      "scr_dir1_threshold_100": 0.38659786529350676,
+      "scr_metric_threshold_100": 0.282258006369044,
+      "scr_dir2_threshold_100": 0.282258006369044,
+      "scr_dir1_threshold_500": 0.37113384007521244,
+      "scr_metric_threshold_500": 0.43951619299405154,
+      "scr_dir2_threshold_500": 0.43951619299405154
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
+      "scr_dir1_threshold_2": 0.20270277526740202,
+      "scr_metric_threshold_2": 0.27111103575907436,
+      "scr_dir2_threshold_2": 0.27111103575907436,
+      "scr_dir1_threshold_5": 0.24774777677362747,
+      "scr_metric_threshold_5": 0.3422222834457521,
+      "scr_dir2_threshold_5": 0.3422222834457521,
+      "scr_dir1_threshold_10": 0.3153152790329657,
+      "scr_metric_threshold_10": 0.4000001059638017,
+      "scr_dir2_threshold_10": 0.4000001059638017,
+      "scr_dir1_threshold_20": 0.3603602805391911,
+      "scr_metric_threshold_20": 0.45777766357234706,
+      "scr_dir2_threshold_20": 0.45777766357234706,
+      "scr_dir1_threshold_50": 0.4189188898930392,
+      "scr_metric_threshold_50": 0.5333332980120661,
+      "scr_dir2_threshold_50": 0.5333332980120661,
+      "scr_dir1_threshold_100": 0.297297224732598,
+      "scr_metric_threshold_100": 0.5466667231806942,
+      "scr_dir2_threshold_100": 0.5466667231806942,
+      "scr_dir1_threshold_500": 0.3153152790329657,
+      "scr_metric_threshold_500": 0.595555507283157,
+      "scr_dir2_threshold_500": 0.595555507283157
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
+      "scr_dir1_threshold_2": 0.060085820441166386,
+      "scr_metric_threshold_2": 0.060085820441166386,
+      "scr_dir2_threshold_2": 0.05714296256601923,
+      "scr_dir1_threshold_5": 0.0643776099411321,
+      "scr_metric_threshold_5": 0.0643776099411321,
+      "scr_dir2_threshold_5": 0.10476195612190681,
+      "scr_dir1_threshold_10": 0.10300422706858779,
+      "scr_metric_threshold_10": 0.10300422706858779,
+      "scr_dir2_threshold_10": 0.14761910708852366,
+      "scr_dir1_threshold_20": 0.15021467900985702,
+      "scr_metric_threshold_20": 0.15021467900985702,
+      "scr_dir2_threshold_20": 0.11904748388971893,
+      "scr_dir1_threshold_50": 0.17596567182353345,
+      "scr_metric_threshold_50": 0.17596567182353345,
+      "scr_dir2_threshold_50": 0.19047625805514054,
+      "scr_dir1_threshold_100": 0.21459228895098914,
+      "scr_metric_threshold_100": 0.21459228895098914,
+      "scr_dir2_threshold_100": 0.19999994323368195,
+      "scr_dir1_threshold_500": 0.21888407845095484,
+      "scr_metric_threshold_500": 0.21888407845095484,
+      "scr_dir2_threshold_500": 0.27142843356751334
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_2",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/scr/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_3_custom_sae_eval_results.json

@@ -0,0 +1,323 @@
+{
+  "eval_type_id": "scr",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": true,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "941954f3-cc39-43ab-951e-ef7dac65494f",
+  "datetime_epoch_millis": 1738802202746,
+  "eval_result_metrics": {
+    "scr_metrics": {
+      "scr_dir1_threshold_2": 0.1998397667116698,
+      "scr_metric_threshold_2": 0.09773468518068165,
+      "scr_dir2_threshold_2": 0.1036206457814039,
+      "scr_dir1_threshold_5": 0.19389433803907274,
+      "scr_metric_threshold_5": 0.1494009315204827,
+      "scr_dir2_threshold_5": 0.15724885650288437,
+      "scr_dir1_threshold_10": 0.20252891396157224,
+      "scr_metric_threshold_10": 0.20638626724773887,
+      "scr_dir2_threshold_10": 0.21464552766444467,
+      "scr_dir1_threshold_20": 0.1869289133935145,
+      "scr_metric_threshold_20": 0.2550003527011395,
+      "scr_dir2_threshold_20": 0.2671248172217763,
+      "scr_dir1_threshold_50": 0.1218689486768515,
+      "scr_metric_threshold_50": 0.29936929010341096,
+      "scr_dir2_threshold_50": 0.3102521588192187,
+      "scr_dir1_threshold_100": 0.0633538494619269,
+      "scr_metric_threshold_100": 0.2931972361072011,
+      "scr_dir2_threshold_100": 0.31354517528118225,
+      "scr_dir1_threshold_500": -0.1510882278136979,
+      "scr_metric_threshold_500": 0.270963096956086,
+      "scr_dir2_threshold_500": 0.290943143416725
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_professor_nurse_results",
+      "scr_dir1_threshold_2": 0.3749997671693945,
+      "scr_metric_threshold_2": 0.012315258641572036,
+      "scr_dir2_threshold_2": 0.012315258641572036,
+      "scr_dir1_threshold_5": 0.39062549476503666,
+      "scr_metric_threshold_5": 0.02709353964956581,
+      "scr_dir2_threshold_5": 0.02709353964956581,
+      "scr_dir1_threshold_10": 0.4062502910382569,
+      "scr_metric_threshold_10": 0.03694577592471611,
+      "scr_dir2_threshold_10": 0.03694577592471611,
+      "scr_dir1_threshold_20": 0.4062502910382569,
+      "scr_metric_threshold_20": 0.051724056932709886,
+      "scr_dir2_threshold_20": 0.051724056932709886,
+      "scr_dir1_threshold_50": 0.3593749708961743,
+      "scr_metric_threshold_50": 0.07635457421585395,
+      "scr_dir2_threshold_50": 0.07635457421585395,
+      "scr_dir1_threshold_100": 0.3125005820765137,
+      "scr_metric_threshold_100": 0.10837430540772658,
+      "scr_dir2_threshold_100": 0.10837430540772658,
+      "scr_dir1_threshold_500": -0.10937450523496334,
+      "scr_metric_threshold_500": 0.06157629320786018,
+      "scr_dir2_threshold_500": 0.06157629320786018
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
+      "scr_dir1_threshold_2": 0.35514009320607276,
+      "scr_metric_threshold_2": 0.16714705376132788,
+      "scr_dir2_threshold_2": 0.16714705376132788,
+      "scr_dir1_threshold_5": 0.2429903262212547,
+      "scr_metric_threshold_5": 0.22190207867332085,
+      "scr_dir2_threshold_5": 0.22190207867332085,
+      "scr_dir1_threshold_10": 0.2056076276124383,
+      "scr_metric_threshold_10": 0.3054755196683302,
+      "scr_dir2_threshold_10": 0.3054755196683302,
+      "scr_dir1_threshold_20": 0.14018679094143036,
+      "scr_metric_threshold_20": 0.36599433085971195,
+      "scr_dir2_threshold_20": 0.36599433085971195,
+      "scr_dir1_threshold_50": 0.04672893031381012,
+      "scr_metric_threshold_50": 0.4092219549842365,
+      "scr_dir2_threshold_50": 0.4092219549842365,
+      "scr_dir1_threshold_100": -0.35514009320607276,
+      "scr_metric_threshold_100": 0.20749287063181265,
+      "scr_dir2_threshold_100": 0.20749287063181265,
+      "scr_dir1_threshold_500": -0.8317756280491677,
+      "scr_metric_threshold_500": 0.18443806905687588,
+      "scr_dir2_threshold_500": 0.18443806905687588
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
+      "scr_dir1_threshold_2": 0.42187508731147705,
+      "scr_metric_threshold_2": 0.017721590807063405,
+      "scr_dir2_threshold_2": 0.017721590807063405,
+      "scr_dir1_threshold_5": 0.43749988358469727,
+      "scr_metric_threshold_5": 0.030379805284471813,
+      "scr_dir2_threshold_5": 0.030379805284471813,
+      "scr_dir1_threshold_10": 0.3749997671693945,
+      "scr_metric_threshold_10": 0.06582283600077923,
+      "scr_dir2_threshold_10": 0.06582283600077923,
+      "scr_dir1_threshold_20": 0.42187508731147705,
+      "scr_metric_threshold_20": 0.11392408119449507,
+      "scr_dir2_threshold_20": 0.11392408119449507,
+      "scr_dir1_threshold_50": 0.18750034924590825,
+      "scr_metric_threshold_50": 0.1544304882404575,
+      "scr_dir2_threshold_50": 0.1544304882404575,
+      "scr_dir1_threshold_100": 0.2968748544808716,
+      "scr_metric_threshold_100": 0.18227852991119212,
+      "scr_dir2_threshold_100": 0.18227852991119212,
+      "scr_dir1_threshold_500": -0.5312495925464404,
+      "scr_metric_threshold_500": 0.13417728471747628,
+      "scr_dir2_threshold_500": 0.13417728471747628
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
+      "scr_dir1_threshold_2": 0.2440945657943584,
+      "scr_metric_threshold_2": 0.12166174206152826,
+      "scr_dir2_threshold_2": 0.12166174206152826,
+      "scr_dir1_threshold_5": 0.1968506228214391,
+      "scr_metric_threshold_5": 0.18100904115762476,
+      "scr_dir2_threshold_5": 0.18100904115762476,
+      "scr_dir1_threshold_10": 0.2519684780685472,
+      "scr_metric_threshold_10": 0.23145402430383724,
+      "scr_dir2_threshold_10": 0.23145402430383724,
+      "scr_dir1_threshold_20": 0.007874381601975306,
+      "scr_metric_threshold_20": 0.2640950830237842,
+      "scr_dir2_threshold_20": 0.2640950830237842,
+      "scr_dir1_threshold_50": -0.13385791664456914,
+      "scr_metric_threshold_50": 0.35014844562765457,
+      "scr_dir2_threshold_50": 0.35014844562765457,
+      "scr_dir1_threshold_100": -0.05511785524710812,
+      "scr_metric_threshold_100": 0.41839770693688383,
+      "scr_dir2_threshold_100": 0.41839770693688383,
+      "scr_dir1_threshold_500": -0.259842390342736,
+      "scr_metric_threshold_500": 0.17210690207611637,
+      "scr_dir2_threshold_500": 0.17210690207611637
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
+      "scr_dir1_threshold_2": 0.03260859706226871,
+      "scr_metric_threshold_2": 0.13725477363090466,
+      "scr_dir2_threshold_2": 0.13725477363090466,
+      "scr_dir1_threshold_5": 0.021739172687599125,
+      "scr_metric_threshold_5": 0.2823528174298009,
+      "scr_dir2_threshold_5": 0.2823528174298009,
+      "scr_dir1_threshold_10": 0.03260859706226871,
+      "scr_metric_threshold_10": 0.41176455463642336,
+      "scr_dir2_threshold_10": 0.41176455463642336,
+      "scr_dir1_threshold_20": 0.05434776974986783,
+      "scr_metric_threshold_20": 0.5254902189944813,
+      "scr_dir2_threshold_20": 0.5254902189944813,
+      "scr_dir1_threshold_50": -0.005434712187334791,
+      "scr_metric_threshold_50": 0.6117645546364233,
+      "scr_dir2_threshold_50": 0.6117645546364233,
+      "scr_dir1_threshold_100": -0.125,
+      "scr_metric_threshold_100": 0.6156863065922739,
+      "scr_dir2_threshold_100": 0.6156863065922739,
+      "scr_dir1_threshold_500": -0.016304460500264333,
+      "scr_metric_threshold_500": 0.643137167820971,
+      "scr_dir2_threshold_500": 0.643137167820971
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
+      "scr_dir1_threshold_2": 0.07731951161061577,
+      "scr_metric_threshold_2": 0.060483807005948444,
+      "scr_dir2_threshold_2": 0.060483807005948444,
+      "scr_dir1_threshold_5": 0.12886585268435963,
+      "scr_metric_threshold_5": 0.12096785435318151,
+      "scr_dir2_threshold_5": 0.12096785435318151,
+      "scr_dir1_threshold_10": 0.15463902322123155,
+      "scr_metric_threshold_10": 0.1491935348194436,
+      "scr_dir2_threshold_10": 0.1491935348194436,
+      "scr_dir1_threshold_20": 0.190721646317109,
+      "scr_metric_threshold_20": 0.1935485188968335,
+      "scr_dir2_threshold_20": 0.1935485188968335,
+      "scr_dir1_threshold_50": 0.18556676641739225,
+      "scr_metric_threshold_50": 0.2661291834404855,
+      "scr_dir2_threshold_50": 0.2661291834404855,
+      "scr_dir1_threshold_100": 0.2010307916356866,
+      "scr_metric_threshold_100": 0.2862904524424683,
+      "scr_dir2_threshold_100": 0.2862904524424683,
+      "scr_dir1_threshold_500": 0.25257713270943044,
+      "scr_metric_threshold_500": 0.3629033227182599,
+      "scr_dir2_threshold_500": 0.3629033227182599
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
+      "scr_dir1_threshold_2": 0.05405389441171559,
+      "scr_metric_threshold_2": 0.22666663840965287,
+      "scr_dir2_threshold_2": 0.22666663840965287,
+      "scr_dir1_threshold_5": 0.0810811101069608,
+      "scr_metric_threshold_5": 0.2800000741746612,
+      "scr_dir2_threshold_5": 0.2800000741746612,
+      "scr_dir1_threshold_10": 0.11261250376556366,
+      "scr_metric_threshold_10": 0.36888886887350414,
+      "scr_dir2_threshold_10": 0.36888886887350414,
+      "scr_dir1_threshold_20": 0.17117111311941172,
+      "scr_metric_threshold_20": 0.42222230463851246,
+      "scr_dir2_threshold_20": 0.42222230463851246,
+      "scr_dir1_threshold_50": 0.27927917043223033,
+      "scr_metric_threshold_50": 0.4711110887409752,
+      "scr_dir2_threshold_50": 0.4711110887409752,
+      "scr_dir1_threshold_100": 0.18018027451428928,
+      "scr_metric_threshold_100": 0.4755554754940165,
+      "scr_dir2_threshold_100": 0.4755554754940165,
+      "scr_dir1_threshold_500": 0.1756755595721568,
+      "scr_metric_threshold_500": 0.4977776741687272,
+      "scr_dir2_threshold_500": 0.4977776741687272
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
+      "scr_dir1_threshold_2": 0.03862661712745569,
+      "scr_metric_threshold_2": 0.03862661712745569,
+      "scr_dir2_threshold_2": 0.08571430193323373,
+      "scr_dir1_threshold_5": 0.05150224144123495,
+      "scr_metric_threshold_5": 0.05150224144123495,
+      "scr_dir2_threshold_5": 0.11428564130044823,
+      "scr_dir1_threshold_10": 0.08154502375487709,
+      "scr_metric_threshold_10": 0.08154502375487709,
+      "scr_dir2_threshold_10": 0.14761910708852366,
+      "scr_dir1_threshold_20": 0.10300422706858779,
+      "scr_metric_threshold_20": 0.10300422706858779,
+      "scr_dir2_threshold_20": 0.19999994323368195,
+      "scr_dir1_threshold_50": 0.05579403094120067,
+      "scr_metric_threshold_50": 0.05579403094120067,
+      "scr_dir2_threshold_50": 0.14285698066766273,
+      "scr_dir1_threshold_100": 0.05150224144123495,
+      "scr_metric_threshold_100": 0.05150224144123495,
+      "scr_dir2_threshold_100": 0.21428575483308432,
+      "scr_dir1_threshold_500": 0.11158806188240133,
+      "scr_metric_threshold_500": 0.11158806188240133,
+      "scr_dir2_threshold_500": 0.27142843356751334
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_3",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/scr/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_4_custom_sae_eval_results.json

@@ -0,0 +1,323 @@
+{
+  "eval_type_id": "scr",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": true,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "c41f7329-1294-48a8-9743-0a7d0c4f653a",
+  "datetime_epoch_millis": 1738803092208,
+  "eval_result_metrics": {
+    "scr_metrics": {
+      "scr_dir1_threshold_2": 0.16662920298711476,
+      "scr_metric_threshold_2": 0.09319447246027145,
+      "scr_dir2_threshold_2": 0.09836765398605983,
+      "scr_dir1_threshold_5": 0.18595827828202774,
+      "scr_metric_threshold_5": 0.14338717270565676,
+      "scr_dir2_threshold_5": 0.14980960699387474,
+      "scr_dir1_threshold_10": 0.142764782657517,
+      "scr_metric_threshold_10": 0.18711664692078173,
+      "scr_dir2_threshold_10": 0.19460437162639055,
+      "scr_dir1_threshold_20": 0.16422745930077481,
+      "scr_metric_threshold_20": 0.22564147974603221,
+      "scr_dir2_threshold_20": 0.23467224389709979,
+      "scr_dir1_threshold_50": 0.07719326484466627,
+      "scr_metric_threshold_50": 0.27652624846177276,
+      "scr_dir2_threshold_50": 0.2778112550108285,
+      "scr_dir1_threshold_100": 0.048861431225419735,
+      "scr_metric_threshold_100": 0.3011672908990718,
+      "scr_dir2_threshold_100": 0.3098454762519983,
+      "scr_dir1_threshold_500": -0.16907412416599507,
+      "scr_metric_threshold_500": 0.2543551795776049,
+      "scr_dir2_threshold_500": 0.26570810838714465
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_professor_nurse_results",
+      "scr_dir1_threshold_2": 0.32812537834973393,
+      "scr_metric_threshold_2": 0.017241303374415515,
+      "scr_dir2_threshold_2": 0.017241303374415515,
+      "scr_dir1_threshold_5": 0.3593749708961743,
+      "scr_metric_threshold_5": 0.03448275355829437,
+      "scr_dir2_threshold_5": 0.03448275355829437,
+      "scr_dir1_threshold_10": 0.3437501746229541,
+      "scr_metric_threshold_10": 0.044334989833444666,
+      "scr_dir2_threshold_10": 0.044334989833444666,
+      "scr_dir1_threshold_20": 0.3437501746229541,
+      "scr_metric_threshold_20": 0.06403931557428191,
+      "scr_dir2_threshold_20": 0.06403931557428191,
+      "scr_dir1_threshold_50": 0.2968748544808716,
+      "scr_metric_threshold_50": 0.07142852948301047,
+      "scr_dir2_threshold_50": 0.07142852948301047,
+      "scr_dir1_threshold_100": 0.2343756693879908,
+      "scr_metric_threshold_100": 0.09852206913257629,
+      "scr_dir2_threshold_100": 0.09852206913257629,
+      "scr_dir1_threshold_500": -0.14062502910382568,
+      "scr_metric_threshold_500": 0.05418707929913162,
+      "scr_dir2_threshold_500": 0.05418707929913162
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
+      "scr_dir1_threshold_2": 0.2897198135878545,
+      "scr_metric_threshold_2": 0.1498558666944707,
+      "scr_dir2_threshold_2": 0.1498558666944707,
+      "scr_dir1_threshold_5": 0.33644874390166457,
+      "scr_metric_threshold_5": 0.21325648513989226,
+      "scr_dir2_threshold_5": 0.21325648513989226,
+      "scr_dir1_threshold_10": 0.16822437195083229,
+      "scr_metric_threshold_10": 0.23631128671482904,
+      "scr_dir2_threshold_10": 0.23631128671482904,
+      "scr_dir1_threshold_20": 0.1588786972986282,
+      "scr_metric_threshold_20": 0.328530321243267,
+      "scr_dir2_threshold_20": 0.328530321243267,
+      "scr_dir1_threshold_50": -0.3177573945972564,
+      "scr_metric_threshold_50": 0.37175794536779155,
+      "scr_dir2_threshold_50": 0.37175794536779155,
+      "scr_dir1_threshold_100": -0.2149533022646424,
+      "scr_metric_threshold_100": 0.4034583404761569,
+      "scr_dir2_threshold_100": 0.4034583404761569,
+      "scr_dir1_threshold_500": -0.9532710696861899,
+      "scr_metric_threshold_500": 0.19596546984434426,
+      "scr_dir2_threshold_500": 0.19596546984434426
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
+      "scr_dir1_threshold_2": 0.3593749708961743,
+      "scr_metric_threshold_2": 0.025316579852636207,
+      "scr_dir2_threshold_2": 0.025316579852636207,
+      "scr_dir1_threshold_5": 0.39062549476503666,
+      "scr_metric_threshold_5": 0.04050640704596242,
+      "scr_dir2_threshold_5": 0.04050640704596242,
+      "scr_dir1_threshold_10": 0.2656252619344312,
+      "scr_metric_threshold_10": 0.06329122328486143,
+      "scr_dir2_threshold_10": 0.06329122328486143,
+      "scr_dir1_threshold_20": 0.3125005820765137,
+      "scr_metric_threshold_20": 0.10379747943300446,
+      "scr_dir2_threshold_20": 0.10379747943300446,
+      "scr_dir1_threshold_50": 0.21874994179234863,
+      "scr_metric_threshold_50": 0.14683549919488467,
+      "scr_dir2_threshold_50": 0.14683549919488467,
+      "scr_dir1_threshold_100": 0.15624982537704588,
+      "scr_metric_threshold_100": 0.1848101426271099,
+      "scr_dir2_threshold_100": 0.1848101426271099,
+      "scr_dir1_threshold_500": -0.4062493597158349,
+      "scr_metric_threshold_500": 0.10379747943300446,
+      "scr_dir2_threshold_500": 0.10379747943300446
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
+      "scr_dir1_threshold_2": 0.1968506228214391,
+      "scr_metric_threshold_2": 0.11572710058610643,
+      "scr_dir2_threshold_2": 0.11572710058610643,
+      "scr_dir1_threshold_5": 0.22834674124598078,
+      "scr_metric_threshold_5": 0.16913958133840545,
+      "scr_dir2_threshold_5": 0.16913958133840545,
+      "scr_dir1_threshold_10": 0.07874006139746104,
+      "scr_metric_threshold_10": 0.22551938282841538,
+      "scr_dir2_threshold_10": 0.22551938282841538,
+      "scr_dir1_threshold_20": 0.08661444299943634,
+      "scr_metric_threshold_20": 0.26112776228607326,
+      "scr_dir2_threshold_20": 0.26112776228607326,
+      "scr_dir1_threshold_50": -0.023621736822566405,
+      "scr_metric_threshold_50": 0.33531166507072435,
+      "scr_dir2_threshold_50": 0.33531166507072435,
+      "scr_dir1_threshold_100": -0.09448788594583864,
+      "scr_metric_threshold_100": 0.3887241458230234,
+      "scr_dir2_threshold_100": 0.3887241458230234,
+      "scr_dir1_threshold_500": -0.2283462719181943,
+      "scr_metric_threshold_500": 0.1899110033707575,
+      "scr_dir2_threshold_500": 0.1899110033707575
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
+      "scr_dir1_threshold_2": 0.027173884874933916,
+      "scr_metric_threshold_2": 0.12549021899448132,
+      "scr_dir2_threshold_2": 0.12549021899448132,
+      "scr_dir1_threshold_5": 0.0380433092496035,
+      "scr_metric_threshold_5": 0.24313716782097103,
+      "scr_dir2_threshold_5": 0.24313716782097103,
+      "scr_dir1_threshold_10": 0.0380433092496035,
+      "scr_metric_threshold_10": 0.3686273868154523,
+      "scr_dir2_threshold_10": 0.3686273868154523,
+      "scr_dir1_threshold_20": 0.06521719412453741,
+      "scr_metric_threshold_20": 0.41176455463642336,
+      "scr_dir2_threshold_20": 0.41176455463642336,
+      "scr_dir1_threshold_50": -0.07065223025013216,
+      "scr_metric_threshold_50": 0.5294117372066224,
+      "scr_dir2_threshold_50": 0.5294117372066224,
+      "scr_dir1_threshold_100": -0.0815219785630617,
+      "scr_metric_threshold_100": 0.5647058686033112,
+      "scr_dir2_threshold_100": 0.5647058686033112,
+      "scr_dir1_threshold_500": -0.03804363318786346,
+      "scr_metric_threshold_500": 0.5882352116198672,
+      "scr_dir2_threshold_500": 0.5882352116198672
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
+      "scr_dir1_threshold_2": 0.05670091373303265,
+      "scr_metric_threshold_2": 0.06854845881151235,
+      "scr_dir2_threshold_2": 0.06854845881151235,
+      "scr_dir1_threshold_5": 0.08247408426990457,
+      "scr_metric_threshold_5": 0.14112912335516434,
+      "scr_dir2_threshold_5": 0.14112912335516434,
+      "scr_dir1_threshold_10": 0.1340204253436484,
+      "scr_metric_threshold_10": 0.16129039235714715,
+      "scr_dir2_threshold_10": 0.16129039235714715,
+      "scr_dir1_threshold_20": 0.14432987790265395,
+      "scr_metric_threshold_20": 0.18145166135912996,
+      "scr_dir2_threshold_20": 0.18145166135912996,
+      "scr_dir1_threshold_50": 0.12886585268435963,
+      "scr_metric_threshold_50": 0.23790326263293876,
+      "scr_dir2_threshold_50": 0.23790326263293876,
+      "scr_dir1_threshold_100": 0.12886585268435963,
+      "scr_metric_threshold_100": 0.2782258006369044,
+      "scr_dir2_threshold_100": 0.2782258006369044,
+      "scr_dir1_threshold_500": 0.1030926821474877,
+      "scr_metric_threshold_500": 0.35483867091269605,
+      "scr_dir2_threshold_500": 0.35483867091269605
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
+      "scr_dir1_threshold_2": 0.045045001506225466,
+      "scr_metric_threshold_2": 0.21333321324102472,
+      "scr_dir2_threshold_2": 0.21333321324102472,
+      "scr_dir1_threshold_5": 0.01801805430036767,
+      "scr_metric_threshold_5": 0.27111103575907436,
+      "scr_dir2_threshold_5": 0.27111103575907436,
+      "scr_dir1_threshold_10": 0.045045001506225466,
+      "scr_metric_threshold_10": 0.32888885827712394,
+      "scr_dir2_threshold_10": 0.32888885827712394,
+      "scr_dir1_threshold_20": 0.1081080573128186,
+      "scr_metric_threshold_20": 0.36000009536742156,
+      "scr_dir2_threshold_20": 0.36000009536742156,
+      "scr_dir1_threshold_50": 0.24774777677362747,
+      "scr_metric_threshold_50": 0.38222229404213226,
+      "scr_dir2_threshold_50": 0.38222229404213226,
+      "scr_dir1_threshold_100": 0.19369361387252446,
+      "scr_metric_threshold_100": 0.42222230463851246,
+      "scr_dir2_threshold_100": 0.42222230463851246,
+      "scr_dir1_threshold_500": 0.22072082956776967,
+      "scr_metric_threshold_500": 0.45777766357234706,
+      "scr_dir2_threshold_500": 0.45777766357234706
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
+      "scr_dir1_threshold_2": 0.030043038127524242,
+      "scr_metric_threshold_2": 0.030043038127524242,
+      "scr_dir2_threshold_2": 0.07142849033383136,
+      "scr_dir1_threshold_5": 0.03433482762748996,
+      "scr_metric_threshold_5": 0.03433482762748996,
+      "scr_dir2_threshold_5": 0.08571430193323373,
+      "scr_dir1_threshold_10": 0.06866965525497992,
+      "scr_metric_threshold_10": 0.06866965525497992,
+      "scr_dir2_threshold_10": 0.12857145289985059,
+      "scr_dir1_threshold_20": 0.09442064806865635,
+      "scr_metric_threshold_20": 0.09442064806865635,
+      "scr_dir2_threshold_20": 0.16666676127719673,
+      "scr_dir1_threshold_50": 0.13733905469607777,
+      "scr_metric_threshold_50": 0.13733905469607777,
+      "scr_dir2_threshold_50": 0.14761910708852366,
+      "scr_dir1_threshold_100": 0.06866965525497992,
+      "scr_metric_threshold_100": 0.06866965525497992,
+      "scr_dir2_threshold_100": 0.13809513807839202,
+      "scr_dir1_threshold_500": 0.09012885856869063,
+      "scr_metric_threshold_500": 0.09012885856869063,
+      "scr_dir2_threshold_500": 0.18095228904500887
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_4",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/scr/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_5_custom_sae_eval_results.json

@@ -0,0 +1,323 @@
+{
+  "eval_type_id": "scr",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": true,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "f172b11d-e73d-4265-bcff-b86b3f4483ba",
+  "datetime_epoch_millis": 1738801762536,
+  "eval_result_metrics": {
+    "scr_metrics": {
+      "scr_dir1_threshold_2": 0.1194317497817165,
+      "scr_metric_threshold_2": 0.07554747805773677,
+      "scr_dir2_threshold_2": 0.08125716524775614,
+      "scr_dir1_threshold_5": 0.14323898317939607,
+      "scr_metric_threshold_5": 0.11980903429955778,
+      "scr_dir2_threshold_5": 0.12575375153644316,
+      "scr_dir1_threshold_10": 0.1363385070457653,
+      "scr_metric_threshold_10": 0.1432143223046469,
+      "scr_dir2_threshold_10": 0.1479021298215021,
+      "scr_dir1_threshold_20": 0.09621199123652632,
+      "scr_metric_threshold_20": 0.16714578684669865,
+      "scr_dir2_threshold_20": 0.18160776541995755,
+      "scr_dir1_threshold_50": 0.009138466559556352,
+      "scr_metric_threshold_50": 0.22111083513489554,
+      "scr_dir2_threshold_50": 0.2409887142151447,
+      "scr_dir1_threshold_100": 0.0010560166167327316,
+      "scr_metric_threshold_100": 0.24574769061347318,
+      "scr_dir2_threshold_100": 0.2606209691420875,
+      "scr_dir1_threshold_500": -0.12225683433428539,
+      "scr_metric_threshold_500": 0.20851088490922143,
+      "scr_dir2_threshold_500": 0.22618408062658932
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_professor_nurse_results",
+      "scr_dir1_threshold_2": 0.18750034924590825,
+      "scr_metric_threshold_2": 0.017241303374415515,
+      "scr_dir2_threshold_2": 0.017241303374415515,
+      "scr_dir1_threshold_5": 0.20312514551912844,
+      "scr_metric_threshold_5": 0.049261034566288144,
+      "scr_dir2_threshold_5": 0.049261034566288144,
+      "scr_dir1_threshold_10": 0.2968748544808716,
+      "scr_metric_threshold_10": 0.06157629320786018,
+      "scr_dir2_threshold_10": 0.06157629320786018,
+      "scr_dir1_threshold_20": 0.20312514551912844,
+      "scr_metric_threshold_20": 0.05911327084143844,
+      "scr_dir2_threshold_20": 0.05911327084143844,
+      "scr_dir1_threshold_50": 0.0937506402841651,
+      "scr_metric_threshold_50": 0.08128076575816078,
+      "scr_dir2_threshold_50": 0.08128076575816078,
+      "scr_dir1_threshold_100": 0.14062502910382568,
+      "scr_metric_threshold_100": 0.08620681049100425,
+      "scr_dir2_threshold_100": 0.08620681049100425,
+      "scr_dir1_threshold_500": -0.28125005820765137,
+      "scr_metric_threshold_500": 0.06403931557428191,
+      "scr_dir2_threshold_500": 0.06403931557428191
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
+      "scr_dir1_threshold_2": 0.2336446515690506,
+      "scr_metric_threshold_2": 0.11239202884933491,
+      "scr_dir2_threshold_2": 0.11239202884933491,
+      "scr_dir1_threshold_5": 0.2710279072306566,
+      "scr_metric_threshold_5": 0.1325648513989227,
+      "scr_dir2_threshold_5": 0.1325648513989227,
+      "scr_dir1_threshold_10": 0.14953246559363445,
+      "scr_metric_threshold_10": 0.16426524650728808,
+      "scr_dir2_threshold_10": 0.16426524650728808,
+      "scr_dir1_threshold_20": 0.16822437195083229,
+      "scr_metric_threshold_20": 0.1757924755234473,
+      "scr_dir2_threshold_20": 0.1757924755234473,
+      "scr_dir1_threshold_50": -0.2056076276124383,
+      "scr_metric_threshold_50": 0.3025937124142904,
+      "scr_dir2_threshold_50": 0.3025937124142904,
+      "scr_dir1_threshold_100": -0.24299088327404436,
+      "scr_metric_threshold_100": 0.328530321243267,
+      "scr_dir2_threshold_100": 0.328530321243267,
+      "scr_dir1_threshold_500": -0.46728986019089086,
+      "scr_metric_threshold_500": 0.21902027141928104,
+      "scr_dir2_threshold_500": 0.21902027141928104
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
+      "scr_dir1_threshold_2": 0.250000465661211,
+      "scr_metric_threshold_2": 0.025316579852636207,
+      "scr_dir2_threshold_2": 0.025316579852636207,
+      "scr_dir1_threshold_5": 0.3125005820765137,
+      "scr_metric_threshold_5": 0.04303801976188022,
+      "scr_dir2_threshold_5": 0.04303801976188022,
+      "scr_dir1_threshold_10": 0.28125005820765137,
+      "scr_metric_threshold_10": 0.055696234239288635,
+      "scr_dir2_threshold_10": 0.055696234239288635,
+      "scr_dir1_threshold_20": 0.1250002328306055,
+      "scr_metric_threshold_20": 0.08860765223967824,
+      "scr_dir2_threshold_20": 0.08860765223967824,
+      "scr_dir1_threshold_50": 0.03125052386886235,
+      "scr_metric_threshold_50": 0.1265822956719035,
+      "scr_dir2_threshold_50": 0.1265822956719035,
+      "scr_dir1_threshold_100": -0.07812491268852294,
+      "scr_metric_threshold_100": 0.15696210095637528,
+      "scr_dir2_threshold_100": 0.15696210095637528,
+      "scr_dir1_threshold_500": -0.3593749708961743,
+      "scr_metric_threshold_500": 0.10126586671708666,
+      "scr_dir2_threshold_500": 0.10126586671708666
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
+      "scr_dir1_threshold_2": 0.14960621052073325,
+      "scr_metric_threshold_2": 0.07715140039073762,
+      "scr_dir2_threshold_2": 0.07715140039073762,
+      "scr_dir1_threshold_5": 0.16535450439689736,
+      "scr_metric_threshold_5": 0.13946602022454502,
+      "scr_dir2_threshold_5": 0.13946602022454502,
+      "scr_dir1_threshold_10": 0.07874006139746104,
+      "scr_metric_threshold_10": 0.15727012151918615,
+      "scr_dir2_threshold_10": 0.15727012151918615,
+      "scr_dir1_threshold_20": 0.07874006139746104,
+      "scr_metric_threshold_20": 0.1988131424522659,
+      "scr_dir2_threshold_20": 0.1988131424522659,
+      "scr_dir1_threshold_50": 0.007874381601975306,
+      "scr_metric_threshold_50": 0.26112776228607326,
+      "scr_dir2_threshold_50": 0.26112776228607326,
+      "scr_dir1_threshold_100": -0.07874006139746104,
+      "scr_metric_threshold_100": 0.2759645428430035,
+      "scr_dir2_threshold_100": 0.2759645428430035,
+      "scr_dir1_threshold_500": -0.18110232894527498,
+      "scr_metric_threshold_500": 0.1364985226184585,
+      "scr_dir2_threshold_500": 0.1364985226184585
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
+      "scr_dir1_threshold_2": 0.027173884874933916,
+      "scr_metric_threshold_2": 0.10588216044635698,
+      "scr_dir2_threshold_2": 0.10588216044635698,
+      "scr_dir1_threshold_5": 0.05434776974986783,
+      "scr_metric_threshold_5": 0.21568630659227384,
+      "scr_dir2_threshold_5": 0.21568630659227384,
+      "scr_dir1_threshold_10": 0.059782481937202626,
+      "scr_metric_threshold_10": 0.262744992625386,
+      "scr_dir2_threshold_10": 0.262744992625386,
+      "scr_dir1_threshold_20": 0.06521719412453741,
+      "scr_metric_threshold_20": 0.3647058686033112,
+      "scr_dir2_threshold_20": 0.3647058686033112,
+      "scr_dir1_threshold_50": -0.027173884874933916,
+      "scr_metric_threshold_50": 0.45490195620110374,
+      "scr_dir2_threshold_50": 0.45490195620110374,
+      "scr_dir1_threshold_100": 0.0,
+      "scr_metric_threshold_100": 0.4823528174298009,
+      "scr_dir2_threshold_100": 0.4823528174298009,
+      "scr_dir1_threshold_500": 0.05434776974986783,
+      "scr_metric_threshold_500": 0.5137254306143486,
+      "scr_dir2_threshold_500": 0.5137254306143486
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
+      "scr_dir1_threshold_2": 0.04123688851473832,
+      "scr_metric_threshold_2": 0.08064531634921589,
+      "scr_dir2_threshold_2": 0.08064531634921589,
+      "scr_dir1_threshold_5": 0.08762865692919337,
+      "scr_metric_threshold_5": 0.10887099681547797,
+      "scr_dir2_threshold_5": 0.10887099681547797,
+      "scr_dir1_threshold_10": 0.0979381094881989,
+      "scr_metric_threshold_10": 0.10483879108333834,
+      "scr_dir2_threshold_10": 0.10483879108333834,
+      "scr_dir1_threshold_20": 0.030927743196160724,
+      "scr_metric_threshold_20": 0.1572581866250075,
+      "scr_dir2_threshold_20": 0.1572581866250075,
+      "scr_dir1_threshold_50": 0.09278322958848217,
+      "scr_metric_threshold_50": 0.20967734182539205,
+      "scr_dir2_threshold_50": 0.20967734182539205,
+      "scr_dir1_threshold_100": 0.13917499800293723,
+      "scr_metric_threshold_100": 0.2419354683650784,
+      "scr_dir2_threshold_100": 0.2419354683650784,
+      "scr_dir1_threshold_500": 0.04639146117402712,
+      "scr_metric_threshold_500": 0.2701613891726251,
+      "scr_dir2_threshold_500": 0.2701613891726251
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
+      "scr_dir1_threshold_2": 0.0405405550534804,
+      "scr_metric_threshold_2": 0.16000004238552068,
+      "scr_dir2_threshold_2": 0.16000004238552068,
+      "scr_dir1_threshold_5": 0.009008892905490125,
+      "scr_metric_threshold_5": 0.22666663840965287,
+      "scr_dir2_threshold_5": 0.22666663840965287,
+      "scr_dir1_threshold_10": 0.045045001506225466,
+      "scr_metric_threshold_10": 0.25777787549995046,
+      "scr_dir2_threshold_10": 0.25777787549995046,
+      "scr_dir1_threshold_20": 0.08558555655970586,
+      "scr_metric_threshold_20": 0.2800000741746612,
+      "scr_dir2_threshold_20": 0.2800000741746612,
+      "scr_dir1_threshold_50": 0.06306305580659313,
+      "scr_metric_threshold_50": 0.3155554331084958,
+      "scr_dir2_threshold_50": 0.3155554331084958,
+      "scr_dir1_threshold_100": 0.08558555655970586,
+      "scr_metric_threshold_100": 0.3511110569518347,
+      "scr_dir2_threshold_100": 0.3511110569518347,
+      "scr_dir1_threshold_500": 0.18018027451428928,
+      "scr_metric_threshold_500": 0.33333324503016526,
+      "scr_dir2_threshold_500": 0.33333324503016526
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
+      "scr_dir1_threshold_2": 0.025750992813676425,
+      "scr_metric_threshold_2": 0.025750992813676425,
+      "scr_dir2_threshold_2": 0.07142849033383136,
+      "scr_dir1_threshold_5": 0.042918406627421406,
+      "scr_metric_threshold_5": 0.042918406627421406,
+      "scr_dir2_threshold_5": 0.09047614452250444,
+      "scr_dir1_threshold_10": 0.08154502375487709,
+      "scr_metric_threshold_10": 0.08154502375487709,
+      "scr_dir2_threshold_10": 0.11904748388971893,
+      "scr_dir1_threshold_20": 0.012875624313779262,
+      "scr_metric_threshold_20": 0.012875624313779262,
+      "scr_dir2_threshold_20": 0.12857145289985059,
+      "scr_dir1_threshold_50": 0.01716741381374498,
+      "scr_metric_threshold_50": 0.01716741381374498,
+      "scr_dir2_threshold_50": 0.17619044645573817,
+      "scr_dir1_threshold_100": 0.042918406627421406,
+      "scr_metric_threshold_100": 0.042918406627421406,
+      "scr_dir2_threshold_100": 0.1619046348563358,
+      "scr_dir1_threshold_500": 0.030043038127524242,
+      "scr_metric_threshold_500": 0.030043038127524242,
+      "scr_dir2_threshold_500": 0.17142860386646747
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_5",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/sparse_probing/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_0_custom_sae_eval_results.json

@@ -0,0 +1,670 @@
+{
+  "eval_type_id": "sparse_probing",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "LabHC/bias_in_bios_class_set2",
+      "LabHC/bias_in_bios_class_set3",
+      "canrager/amazon_reviews_mcauley_1and5",
+      "canrager/amazon_reviews_mcauley_1and5_sentiment",
+      "codeparrot/github-code",
+      "fancyzhx/ag_news",
+      "Helsinki-NLP/europarl"
+    ],
+    "probe_train_set_size": 4000,
+    "probe_test_set_size": 1000,
+    "context_length": 128,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "model_name": "gemma-2-2b",
+    "k_values": [
+      1,
+      2,
+      5
+    ],
+    "lower_vram_usage": false
+  },
+  "eval_id": "1d2f727d-ba91-4f28-ae1e-d5e769b99804",
+  "datetime_epoch_millis": 1738809581267,
+  "eval_result_metrics": {
+    "llm": {
+      "llm_test_accuracy": 0.9598687920719384,
+      "llm_top_1_test_accuracy": 0.6589812500000001,
+      "llm_top_2_test_accuracy": 0.7174812500000001,
+      "llm_top_5_test_accuracy": 0.7826062500000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null
+    },
+    "sae": {
+      "sae_test_accuracy": 0.959237540513277,
+      "sae_top_1_test_accuracy": 0.7985375,
+      "sae_top_2_test_accuracy": 0.83240625,
+      "sae_top_5_test_accuracy": 0.8882062500000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_results",
+      "llm_test_accuracy": 0.9694000363349915,
+      "llm_top_1_test_accuracy": 0.6436000000000001,
+      "llm_top_2_test_accuracy": 0.6874,
+      "llm_top_5_test_accuracy": 0.7908,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9688000321388245,
+      "sae_top_1_test_accuracy": 0.8117999999999999,
+      "sae_top_2_test_accuracy": 0.8220000000000001,
+      "sae_top_5_test_accuracy": 0.8633999999999998,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set2_results",
+      "llm_test_accuracy": 0.9544000387191772,
+      "llm_top_1_test_accuracy": 0.67,
+      "llm_top_2_test_accuracy": 0.7148,
+      "llm_top_5_test_accuracy": 0.7716,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9506000518798828,
+      "sae_top_1_test_accuracy": 0.7866,
+      "sae_top_2_test_accuracy": 0.8413999999999999,
+      "sae_top_5_test_accuracy": 0.8710000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set3_results",
+      "llm_test_accuracy": 0.9320000410079956,
+      "llm_top_1_test_accuracy": 0.6918,
+      "llm_top_2_test_accuracy": 0.7338,
+      "llm_top_5_test_accuracy": 0.765,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9304000496864319,
+      "sae_top_1_test_accuracy": 0.8116,
+      "sae_top_2_test_accuracy": 0.8392000000000002,
+      "sae_top_5_test_accuracy": 0.8754,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_results",
+      "llm_test_accuracy": 0.9196000456809997,
+      "llm_top_1_test_accuracy": 0.6048,
+      "llm_top_2_test_accuracy": 0.6406000000000001,
+      "llm_top_5_test_accuracy": 0.6696,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9196000337600708,
+      "sae_top_1_test_accuracy": 0.7326,
+      "sae_top_2_test_accuracy": 0.776,
+      "sae_top_5_test_accuracy": 0.8576,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_sentiment_results",
+      "llm_test_accuracy": 0.9820000529289246,
+      "llm_top_1_test_accuracy": 0.672,
+      "llm_top_2_test_accuracy": 0.724,
+      "llm_top_5_test_accuracy": 0.766,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9805000424385071,
+      "sae_top_1_test_accuracy": 0.733,
+      "sae_top_2_test_accuracy": 0.773,
+      "sae_top_5_test_accuracy": 0.909,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "codeparrot/github-code_results",
+      "llm_test_accuracy": 0.969200050830841,
+      "llm_top_1_test_accuracy": 0.6428,
+      "llm_top_2_test_accuracy": 0.6920000000000001,
+      "llm_top_5_test_accuracy": 0.7656000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9714000463485718,
+      "sae_top_1_test_accuracy": 0.7604,
+      "sae_top_2_test_accuracy": 0.8074,
+      "sae_top_5_test_accuracy": 0.8694000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "fancyzhx/ag_news_results",
+      "llm_test_accuracy": 0.9527500420808792,
+      "llm_top_1_test_accuracy": 0.69725,
+      "llm_top_2_test_accuracy": 0.76625,
+      "llm_top_5_test_accuracy": 0.8192499999999999,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9530000388622284,
+      "sae_top_1_test_accuracy": 0.8065,
+      "sae_top_2_test_accuracy": 0.8452500000000001,
+      "sae_top_5_test_accuracy": 0.86225,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "Helsinki-NLP/europarl_results",
+      "llm_test_accuracy": 0.9996000289916992,
+      "llm_top_1_test_accuracy": 0.6496000000000001,
+      "llm_top_2_test_accuracy": 0.7809999999999999,
+      "llm_top_5_test_accuracy": 0.913,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9996000289916992,
+      "sae_top_1_test_accuracy": 0.9458,
+      "sae_top_2_test_accuracy": 0.9549999999999998,
+      "sae_top_5_test_accuracy": 0.9975999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_0",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1_results": {
+      "sae_test_accuracy": {
+        "0": 0.956000030040741,
+        "1": 0.968000054359436,
+        "2": 0.9570000171661377,
+        "6": 0.9880000352859497,
+        "9": 0.9750000238418579
+      },
+      "llm_test_accuracy": {
+        "0": 0.9510000348091125,
+        "1": 0.9670000672340393,
+        "2": 0.9520000219345093,
+        "6": 0.9930000305175781,
+        "9": 0.984000027179718
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.568,
+        "1": 0.629,
+        "2": 0.679,
+        "6": 0.791,
+        "9": 0.551
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.585,
+        "1": 0.666,
+        "2": 0.673,
+        "6": 0.801,
+        "9": 0.712
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.72,
+        "1": 0.707,
+        "2": 0.764,
+        "6": 0.899,
+        "9": 0.864
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.625,
+        "1": 0.663,
+        "2": 0.863,
+        "6": 0.973,
+        "9": 0.935
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.66,
+        "1": 0.665,
+        "2": 0.863,
+        "6": 0.983,
+        "9": 0.939
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.741,
+        "1": 0.771,
+        "2": 0.876,
+        "6": 0.986,
+        "9": 0.943
+      }
+    },
+    "LabHC/bias_in_bios_class_set2_results": {
+      "sae_test_accuracy": {
+        "11": 0.9650000333786011,
+        "13": 0.9490000605583191,
+        "14": 0.9540000557899475,
+        "18": 0.9300000667572021,
+        "19": 0.9550000429153442
+      },
+      "llm_test_accuracy": {
+        "11": 0.971000075340271,
+        "13": 0.9520000219345093,
+        "14": 0.956000030040741,
+        "18": 0.9360000491142273,
+        "19": 0.9570000171661377
+      },
+      "llm_top_1_test_accuracy": {
+        "11": 0.561,
+        "13": 0.672,
+        "14": 0.631,
+        "18": 0.7,
+        "19": 0.786
+      },
+      "llm_top_2_test_accuracy": {
+        "11": 0.69,
+        "13": 0.72,
+        "14": 0.677,
+        "18": 0.721,
+        "19": 0.766
+      },
+      "llm_top_5_test_accuracy": {
+        "11": 0.776,
+        "13": 0.742,
+        "14": 0.768,
+        "18": 0.731,
+        "19": 0.841
+      },
+      "sae_top_1_test_accuracy": {
+        "11": 0.729,
+        "13": 0.751,
+        "14": 0.878,
+        "18": 0.732,
+        "19": 0.843
+      },
+      "sae_top_2_test_accuracy": {
+        "11": 0.857,
+        "13": 0.773,
+        "14": 0.88,
+        "18": 0.855,
+        "19": 0.842
+      },
+      "sae_top_5_test_accuracy": {
+        "11": 0.933,
+        "13": 0.831,
+        "14": 0.888,
+        "18": 0.856,
+        "19": 0.847
+      }
+    },
+    "LabHC/bias_in_bios_class_set3_results": {
+      "sae_test_accuracy": {
+        "20": 0.9530000686645508,
+        "21": 0.9290000200271606,
+        "22": 0.9140000343322754,
+        "25": 0.968000054359436,
+        "26": 0.8880000710487366
+      },
+      "llm_test_accuracy": {
+        "20": 0.956000030040741,
+        "21": 0.9260000586509705,
+        "22": 0.9170000553131104,
+        "25": 0.9640000462532043,
+        "26": 0.8970000147819519
+      },
+      "llm_top_1_test_accuracy": {
+        "20": 0.699,
+        "21": 0.772,
+        "22": 0.641,
+        "25": 0.703,
+        "26": 0.644
+      },
+      "llm_top_2_test_accuracy": {
+        "20": 0.811,
+        "21": 0.757,
+        "22": 0.655,
+        "25": 0.762,
+        "26": 0.684
+      },
+      "llm_top_5_test_accuracy": {
+        "20": 0.861,
+        "21": 0.788,
+        "22": 0.712,
+        "25": 0.796,
+        "26": 0.668
+      },
+      "sae_top_1_test_accuracy": {
+        "20": 0.856,
+        "21": 0.77,
+        "22": 0.844,
+        "25": 0.881,
+        "26": 0.707
+      },
+      "sae_top_2_test_accuracy": {
+        "20": 0.899,
+        "21": 0.805,
+        "22": 0.846,
+        "25": 0.886,
+        "26": 0.76
+      },
+      "sae_top_5_test_accuracy": {
+        "20": 0.92,
+        "21": 0.848,
+        "22": 0.891,
+        "25": 0.891,
+        "26": 0.827
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_results": {
+      "sae_test_accuracy": {
+        "1": 0.9570000171661377,
+        "2": 0.9410000443458557,
+        "3": 0.9240000247955322,
+        "5": 0.909000039100647,
+        "6": 0.8670000433921814
+      },
+      "llm_test_accuracy": {
+        "1": 0.956000030040741,
+        "2": 0.937000036239624,
+        "3": 0.9160000681877136,
+        "5": 0.9170000553131104,
+        "6": 0.8720000386238098
+      },
+      "llm_top_1_test_accuracy": {
+        "1": 0.665,
+        "2": 0.596,
+        "3": 0.599,
+        "5": 0.576,
+        "6": 0.588
+      },
+      "llm_top_2_test_accuracy": {
+        "1": 0.73,
+        "2": 0.632,
+        "3": 0.617,
+        "5": 0.615,
+        "6": 0.609
+      },
+      "llm_top_5_test_accuracy": {
+        "1": 0.758,
+        "2": 0.646,
+        "3": 0.627,
+        "5": 0.646,
+        "6": 0.671
+      },
+      "sae_top_1_test_accuracy": {
+        "1": 0.845,
+        "2": 0.669,
+        "3": 0.668,
+        "5": 0.717,
+        "6": 0.764
+      },
+      "sae_top_2_test_accuracy": {
+        "1": 0.876,
+        "2": 0.698,
+        "3": 0.672,
+        "5": 0.856,
+        "6": 0.778
+      },
+      "sae_top_5_test_accuracy": {
+        "1": 0.91,
+        "2": 0.907,
+        "3": 0.802,
+        "5": 0.879,
+        "6": 0.79
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_sentiment_results": {
+      "sae_test_accuracy": {
+        "1.0": 0.9810000658035278,
+        "5.0": 0.9800000190734863
+      },
+      "llm_test_accuracy": {
+        "1.0": 0.9820000529289246,
+        "5.0": 0.9820000529289246
+      },
+      "llm_top_1_test_accuracy": {
+        "1.0": 0.672,
+        "5.0": 0.672
+      },
+      "llm_top_2_test_accuracy": {
+        "1.0": 0.724,
+        "5.0": 0.724
+      },
+      "llm_top_5_test_accuracy": {
+        "1.0": 0.766,
+        "5.0": 0.766
+      },
+      "sae_top_1_test_accuracy": {
+        "1.0": 0.733,
+        "5.0": 0.733
+      },
+      "sae_top_2_test_accuracy": {
+        "1.0": 0.773,
+        "5.0": 0.773
+      },
+      "sae_top_5_test_accuracy": {
+        "1.0": 0.909,
+        "5.0": 0.909
+      }
+    },
+    "codeparrot/github-code_results": {
+      "sae_test_accuracy": {
+        "C": 0.9550000429153442,
+        "Python": 0.9880000352859497,
+        "HTML": 0.9890000224113464,
+        "Java": 0.9670000672340393,
+        "PHP": 0.9580000638961792
+      },
+      "llm_test_accuracy": {
+        "C": 0.9530000686645508,
+        "Python": 0.9860000610351562,
+        "HTML": 0.9880000352859497,
+        "Java": 0.9640000462532043,
+        "PHP": 0.9550000429153442
+      },
+      "llm_top_1_test_accuracy": {
+        "C": 0.657,
+        "Python": 0.637,
+        "HTML": 0.714,
+        "Java": 0.624,
+        "PHP": 0.582
+      },
+      "llm_top_2_test_accuracy": {
+        "C": 0.658,
+        "Python": 0.664,
+        "HTML": 0.801,
+        "Java": 0.697,
+        "PHP": 0.64
+      },
+      "llm_top_5_test_accuracy": {
+        "C": 0.752,
+        "Python": 0.726,
+        "HTML": 0.928,
+        "Java": 0.728,
+        "PHP": 0.694
+      },
+      "sae_top_1_test_accuracy": {
+        "C": 0.611,
+        "Python": 0.934,
+        "HTML": 0.68,
+        "Java": 0.655,
+        "PHP": 0.922
+      },
+      "sae_top_2_test_accuracy": {
+        "C": 0.641,
+        "Python": 0.931,
+        "HTML": 0.887,
+        "Java": 0.655,
+        "PHP": 0.923
+      },
+      "sae_top_5_test_accuracy": {
+        "C": 0.756,
+        "Python": 0.937,
+        "HTML": 0.93,
+        "Java": 0.805,
+        "PHP": 0.919
+      }
+    },
+    "fancyzhx/ag_news_results": {
+      "sae_test_accuracy": {
+        "0": 0.9380000233650208,
+        "1": 0.984000027179718,
+        "2": 0.9360000491142273,
+        "3": 0.9540000557899475
+      },
+      "llm_test_accuracy": {
+        "0": 0.9380000233650208,
+        "1": 0.9880000352859497,
+        "2": 0.9300000667572021,
+        "3": 0.9550000429153442
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.805,
+        "1": 0.67,
+        "2": 0.648,
+        "3": 0.666
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.794,
+        "1": 0.795,
+        "2": 0.686,
+        "3": 0.79
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.818,
+        "1": 0.867,
+        "2": 0.756,
+        "3": 0.836
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.86,
+        "1": 0.958,
+        "2": 0.801,
+        "3": 0.607
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.859,
+        "1": 0.965,
+        "2": 0.833,
+        "3": 0.724
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.871,
+        "1": 0.964,
+        "2": 0.835,
+        "3": 0.779
+      }
+    },
+    "Helsinki-NLP/europarl_results": {
+      "sae_test_accuracy": {
+        "en": 0.999000072479248,
+        "fr": 1.0,
+        "de": 1.0,
+        "es": 1.0,
+        "nl": 0.999000072479248
+      },
+      "llm_test_accuracy": {
+        "en": 1.0,
+        "fr": 1.0,
+        "de": 0.999000072479248,
+        "es": 1.0,
+        "nl": 0.999000072479248
+      },
+      "llm_top_1_test_accuracy": {
+        "en": 0.748,
+        "fr": 0.59,
+        "de": 0.754,
+        "es": 0.494,
+        "nl": 0.662
+      },
+      "llm_top_2_test_accuracy": {
+        "en": 0.822,
+        "fr": 0.596,
+        "de": 0.831,
+        "es": 0.91,
+        "nl": 0.746
+      },
+      "llm_top_5_test_accuracy": {
+        "en": 0.887,
+        "fr": 0.922,
+        "de": 0.909,
+        "es": 0.982,
+        "nl": 0.865
+      },
+      "sae_top_1_test_accuracy": {
+        "en": 0.995,
+        "fr": 0.993,
+        "de": 0.903,
+        "es": 0.92,
+        "nl": 0.918
+      },
+      "sae_top_2_test_accuracy": {
+        "en": 0.997,
+        "fr": 0.992,
+        "de": 0.908,
+        "es": 0.944,
+        "nl": 0.934
+      },
+      "sae_top_5_test_accuracy": {
+        "en": 0.999,
+        "fr": 0.998,
+        "de": 0.995,
+        "es": 0.997,
+        "nl": 0.999
+      }
+    }
+  }
+}

old_relu_eval_results/sparse_probing/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_1_custom_sae_eval_results.json

@@ -0,0 +1,670 @@
+{
+  "eval_type_id": "sparse_probing",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "LabHC/bias_in_bios_class_set2",
+      "LabHC/bias_in_bios_class_set3",
+      "canrager/amazon_reviews_mcauley_1and5",
+      "canrager/amazon_reviews_mcauley_1and5_sentiment",
+      "codeparrot/github-code",
+      "fancyzhx/ag_news",
+      "Helsinki-NLP/europarl"
+    ],
+    "probe_train_set_size": 4000,
+    "probe_test_set_size": 1000,
+    "context_length": 128,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "model_name": "gemma-2-2b",
+    "k_values": [
+      1,
+      2,
+      5
+    ],
+    "lower_vram_usage": false
+  },
+  "eval_id": "12d3597f-e01f-4acd-b4e4-3916a8253103",
+  "datetime_epoch_millis": 1738809822946,
+  "eval_result_metrics": {
+    "llm": {
+      "llm_test_accuracy": 0.9598687920719384,
+      "llm_top_1_test_accuracy": 0.6589812500000001,
+      "llm_top_2_test_accuracy": 0.7174812500000001,
+      "llm_top_5_test_accuracy": 0.7826062500000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null
+    },
+    "sae": {
+      "sae_test_accuracy": 0.9591187916696072,
+      "sae_top_1_test_accuracy": 0.7973625000000001,
+      "sae_top_2_test_accuracy": 0.8503124999999999,
+      "sae_top_5_test_accuracy": 0.8916437500000002,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_results",
+      "llm_test_accuracy": 0.9694000363349915,
+      "llm_top_1_test_accuracy": 0.6436000000000001,
+      "llm_top_2_test_accuracy": 0.6874,
+      "llm_top_5_test_accuracy": 0.7908,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9698000431060791,
+      "sae_top_1_test_accuracy": 0.8183999999999999,
+      "sae_top_2_test_accuracy": 0.8507999999999999,
+      "sae_top_5_test_accuracy": 0.8998000000000002,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set2_results",
+      "llm_test_accuracy": 0.9544000387191772,
+      "llm_top_1_test_accuracy": 0.67,
+      "llm_top_2_test_accuracy": 0.7148,
+      "llm_top_5_test_accuracy": 0.7716,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9524000406265258,
+      "sae_top_1_test_accuracy": 0.7678,
+      "sae_top_2_test_accuracy": 0.7921999999999999,
+      "sae_top_5_test_accuracy": 0.885,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set3_results",
+      "llm_test_accuracy": 0.9320000410079956,
+      "llm_top_1_test_accuracy": 0.6918,
+      "llm_top_2_test_accuracy": 0.7338,
+      "llm_top_5_test_accuracy": 0.765,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9310000538825989,
+      "sae_top_1_test_accuracy": 0.8183999999999999,
+      "sae_top_2_test_accuracy": 0.8488,
+      "sae_top_5_test_accuracy": 0.8701999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_results",
+      "llm_test_accuracy": 0.9196000456809997,
+      "llm_top_1_test_accuracy": 0.6048,
+      "llm_top_2_test_accuracy": 0.6406000000000001,
+      "llm_top_5_test_accuracy": 0.6696,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9168000459671021,
+      "sae_top_1_test_accuracy": 0.737,
+      "sae_top_2_test_accuracy": 0.7807999999999999,
+      "sae_top_5_test_accuracy": 0.8092,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_sentiment_results",
+      "llm_test_accuracy": 0.9820000529289246,
+      "llm_top_1_test_accuracy": 0.672,
+      "llm_top_2_test_accuracy": 0.724,
+      "llm_top_5_test_accuracy": 0.766,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9775000512599945,
+      "sae_top_1_test_accuracy": 0.773,
+      "sae_top_2_test_accuracy": 0.903,
+      "sae_top_5_test_accuracy": 0.93,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "codeparrot/github-code_results",
+      "llm_test_accuracy": 0.969200050830841,
+      "llm_top_1_test_accuracy": 0.6428,
+      "llm_top_2_test_accuracy": 0.6920000000000001,
+      "llm_top_5_test_accuracy": 0.7656000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9718000411987304,
+      "sae_top_1_test_accuracy": 0.7976000000000001,
+      "sae_top_2_test_accuracy": 0.8288,
+      "sae_top_5_test_accuracy": 0.8718,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "fancyzhx/ag_news_results",
+      "llm_test_accuracy": 0.9527500420808792,
+      "llm_top_1_test_accuracy": 0.69725,
+      "llm_top_2_test_accuracy": 0.76625,
+      "llm_top_5_test_accuracy": 0.8192499999999999,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9542500376701355,
+      "sae_top_1_test_accuracy": 0.7125,
+      "sae_top_2_test_accuracy": 0.8314999999999999,
+      "sae_top_5_test_accuracy": 0.87075,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "Helsinki-NLP/europarl_results",
+      "llm_test_accuracy": 0.9996000289916992,
+      "llm_top_1_test_accuracy": 0.6496000000000001,
+      "llm_top_2_test_accuracy": 0.7809999999999999,
+      "llm_top_5_test_accuracy": 0.913,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9994000196456909,
+      "sae_top_1_test_accuracy": 0.9542000000000002,
+      "sae_top_2_test_accuracy": 0.9666,
+      "sae_top_5_test_accuracy": 0.9964000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_1",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1_results": {
+      "sae_test_accuracy": {
+        "0": 0.9540000557899475,
+        "1": 0.9700000286102295,
+        "2": 0.9550000429153442,
+        "6": 0.9910000562667847,
+        "9": 0.9790000319480896
+      },
+      "llm_test_accuracy": {
+        "0": 0.9510000348091125,
+        "1": 0.9670000672340393,
+        "2": 0.9520000219345093,
+        "6": 0.9930000305175781,
+        "9": 0.984000027179718
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.568,
+        "1": 0.629,
+        "2": 0.679,
+        "6": 0.791,
+        "9": 0.551
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.585,
+        "1": 0.666,
+        "2": 0.673,
+        "6": 0.801,
+        "9": 0.712
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.72,
+        "1": 0.707,
+        "2": 0.764,
+        "6": 0.899,
+        "9": 0.864
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.624,
+        "1": 0.693,
+        "2": 0.866,
+        "6": 0.98,
+        "9": 0.929
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.633,
+        "1": 0.819,
+        "2": 0.884,
+        "6": 0.981,
+        "9": 0.937
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.799,
+        "1": 0.878,
+        "2": 0.888,
+        "6": 0.986,
+        "9": 0.948
+      }
+    },
+    "LabHC/bias_in_bios_class_set2_results": {
+      "sae_test_accuracy": {
+        "11": 0.9700000286102295,
+        "13": 0.9390000700950623,
+        "14": 0.9500000476837158,
+        "18": 0.9380000233650208,
+        "19": 0.9650000333786011
+      },
+      "llm_test_accuracy": {
+        "11": 0.971000075340271,
+        "13": 0.9520000219345093,
+        "14": 0.956000030040741,
+        "18": 0.9360000491142273,
+        "19": 0.9570000171661377
+      },
+      "llm_top_1_test_accuracy": {
+        "11": 0.561,
+        "13": 0.672,
+        "14": 0.631,
+        "18": 0.7,
+        "19": 0.786
+      },
+      "llm_top_2_test_accuracy": {
+        "11": 0.69,
+        "13": 0.72,
+        "14": 0.677,
+        "18": 0.721,
+        "19": 0.766
+      },
+      "llm_top_5_test_accuracy": {
+        "11": 0.776,
+        "13": 0.742,
+        "14": 0.768,
+        "18": 0.731,
+        "19": 0.841
+      },
+      "sae_top_1_test_accuracy": {
+        "11": 0.731,
+        "13": 0.692,
+        "14": 0.858,
+        "18": 0.729,
+        "19": 0.829
+      },
+      "sae_top_2_test_accuracy": {
+        "11": 0.844,
+        "13": 0.68,
+        "14": 0.861,
+        "18": 0.731,
+        "19": 0.845
+      },
+      "sae_top_5_test_accuracy": {
+        "11": 0.951,
+        "13": 0.863,
+        "14": 0.88,
+        "18": 0.884,
+        "19": 0.847
+      }
+    },
+    "LabHC/bias_in_bios_class_set3_results": {
+      "sae_test_accuracy": {
+        "20": 0.9620000720024109,
+        "21": 0.9230000376701355,
+        "22": 0.9220000505447388,
+        "25": 0.9590000510215759,
+        "26": 0.8890000581741333
+      },
+      "llm_test_accuracy": {
+        "20": 0.956000030040741,
+        "21": 0.9260000586509705,
+        "22": 0.9170000553131104,
+        "25": 0.9640000462532043,
+        "26": 0.8970000147819519
+      },
+      "llm_top_1_test_accuracy": {
+        "20": 0.699,
+        "21": 0.772,
+        "22": 0.641,
+        "25": 0.703,
+        "26": 0.644
+      },
+      "llm_top_2_test_accuracy": {
+        "20": 0.811,
+        "21": 0.757,
+        "22": 0.655,
+        "25": 0.762,
+        "26": 0.684
+      },
+      "llm_top_5_test_accuracy": {
+        "20": 0.861,
+        "21": 0.788,
+        "22": 0.712,
+        "25": 0.796,
+        "26": 0.668
+      },
+      "sae_top_1_test_accuracy": {
+        "20": 0.881,
+        "21": 0.76,
+        "22": 0.862,
+        "25": 0.88,
+        "26": 0.709
+      },
+      "sae_top_2_test_accuracy": {
+        "20": 0.898,
+        "21": 0.792,
+        "22": 0.897,
+        "25": 0.891,
+        "26": 0.766
+      },
+      "sae_top_5_test_accuracy": {
+        "20": 0.938,
+        "21": 0.842,
+        "22": 0.889,
+        "25": 0.877,
+        "26": 0.805
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_results": {
+      "sae_test_accuracy": {
+        "1": 0.9570000171661377,
+        "2": 0.9300000667572021,
+        "3": 0.9110000729560852,
+        "5": 0.9120000600814819,
+        "6": 0.8740000128746033
+      },
+      "llm_test_accuracy": {
+        "1": 0.956000030040741,
+        "2": 0.937000036239624,
+        "3": 0.9160000681877136,
+        "5": 0.9170000553131104,
+        "6": 0.8720000386238098
+      },
+      "llm_top_1_test_accuracy": {
+        "1": 0.665,
+        "2": 0.596,
+        "3": 0.599,
+        "5": 0.576,
+        "6": 0.588
+      },
+      "llm_top_2_test_accuracy": {
+        "1": 0.73,
+        "2": 0.632,
+        "3": 0.617,
+        "5": 0.615,
+        "6": 0.609
+      },
+      "llm_top_5_test_accuracy": {
+        "1": 0.758,
+        "2": 0.646,
+        "3": 0.627,
+        "5": 0.646,
+        "6": 0.671
+      },
+      "sae_top_1_test_accuracy": {
+        "1": 0.85,
+        "2": 0.751,
+        "3": 0.697,
+        "5": 0.662,
+        "6": 0.725
+      },
+      "sae_top_2_test_accuracy": {
+        "1": 0.858,
+        "2": 0.806,
+        "3": 0.696,
+        "5": 0.776,
+        "6": 0.768
+      },
+      "sae_top_5_test_accuracy": {
+        "1": 0.901,
+        "2": 0.852,
+        "3": 0.73,
+        "5": 0.807,
+        "6": 0.756
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_sentiment_results": {
+      "sae_test_accuracy": {
+        "1.0": 0.9770000576972961,
+        "5.0": 0.9780000448226929
+      },
+      "llm_test_accuracy": {
+        "1.0": 0.9820000529289246,
+        "5.0": 0.9820000529289246
+      },
+      "llm_top_1_test_accuracy": {
+        "1.0": 0.672,
+        "5.0": 0.672
+      },
+      "llm_top_2_test_accuracy": {
+        "1.0": 0.724,
+        "5.0": 0.724
+      },
+      "llm_top_5_test_accuracy": {
+        "1.0": 0.766,
+        "5.0": 0.766
+      },
+      "sae_top_1_test_accuracy": {
+        "1.0": 0.773,
+        "5.0": 0.773
+      },
+      "sae_top_2_test_accuracy": {
+        "1.0": 0.903,
+        "5.0": 0.903
+      },
+      "sae_top_5_test_accuracy": {
+        "1.0": 0.93,
+        "5.0": 0.93
+      }
+    },
+    "codeparrot/github-code_results": {
+      "sae_test_accuracy": {
+        "C": 0.9600000381469727,
+        "Python": 0.9900000691413879,
+        "HTML": 0.9880000352859497,
+        "Java": 0.9640000462532043,
+        "PHP": 0.9570000171661377
+      },
+      "llm_test_accuracy": {
+        "C": 0.9530000686645508,
+        "Python": 0.9860000610351562,
+        "HTML": 0.9880000352859497,
+        "Java": 0.9640000462532043,
+        "PHP": 0.9550000429153442
+      },
+      "llm_top_1_test_accuracy": {
+        "C": 0.657,
+        "Python": 0.637,
+        "HTML": 0.714,
+        "Java": 0.624,
+        "PHP": 0.582
+      },
+      "llm_top_2_test_accuracy": {
+        "C": 0.658,
+        "Python": 0.664,
+        "HTML": 0.801,
+        "Java": 0.697,
+        "PHP": 0.64
+      },
+      "llm_top_5_test_accuracy": {
+        "C": 0.752,
+        "Python": 0.726,
+        "HTML": 0.928,
+        "Java": 0.728,
+        "PHP": 0.694
+      },
+      "sae_top_1_test_accuracy": {
+        "C": 0.615,
+        "Python": 0.93,
+        "HTML": 0.881,
+        "Java": 0.647,
+        "PHP": 0.915
+      },
+      "sae_top_2_test_accuracy": {
+        "C": 0.734,
+        "Python": 0.935,
+        "HTML": 0.906,
+        "Java": 0.656,
+        "PHP": 0.913
+      },
+      "sae_top_5_test_accuracy": {
+        "C": 0.738,
+        "Python": 0.957,
+        "HTML": 0.958,
+        "Java": 0.791,
+        "PHP": 0.915
+      }
+    },
+    "fancyzhx/ag_news_results": {
+      "sae_test_accuracy": {
+        "0": 0.9360000491142273,
+        "1": 0.987000048160553,
+        "2": 0.9340000152587891,
+        "3": 0.9600000381469727
+      },
+      "llm_test_accuracy": {
+        "0": 0.9380000233650208,
+        "1": 0.9880000352859497,
+        "2": 0.9300000667572021,
+        "3": 0.9550000429153442
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.805,
+        "1": 0.67,
+        "2": 0.648,
+        "3": 0.666
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.794,
+        "1": 0.795,
+        "2": 0.686,
+        "3": 0.79
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.818,
+        "1": 0.867,
+        "2": 0.756,
+        "3": 0.836
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.849,
+        "1": 0.637,
+        "2": 0.739,
+        "3": 0.625
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.853,
+        "1": 0.858,
+        "2": 0.824,
+        "3": 0.791
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.866,
+        "1": 0.971,
+        "2": 0.837,
+        "3": 0.809
+      }
+    },
+    "Helsinki-NLP/europarl_results": {
+      "sae_test_accuracy": {
+        "en": 1.0,
+        "fr": 1.0,
+        "de": 1.0,
+        "es": 0.9980000257492065,
+        "nl": 0.999000072479248
+      },
+      "llm_test_accuracy": {
+        "en": 1.0,
+        "fr": 1.0,
+        "de": 0.999000072479248,
+        "es": 1.0,
+        "nl": 0.999000072479248
+      },
+      "llm_top_1_test_accuracy": {
+        "en": 0.748,
+        "fr": 0.59,
+        "de": 0.754,
+        "es": 0.494,
+        "nl": 0.662
+      },
+      "llm_top_2_test_accuracy": {
+        "en": 0.822,
+        "fr": 0.596,
+        "de": 0.831,
+        "es": 0.91,
+        "nl": 0.746
+      },
+      "llm_top_5_test_accuracy": {
+        "en": 0.887,
+        "fr": 0.922,
+        "de": 0.909,
+        "es": 0.982,
+        "nl": 0.865
+      },
+      "sae_top_1_test_accuracy": {
+        "en": 1.0,
+        "fr": 0.991,
+        "de": 0.919,
+        "es": 0.937,
+        "nl": 0.924
+      },
+      "sae_top_2_test_accuracy": {
+        "en": 0.996,
+        "fr": 0.988,
+        "de": 0.913,
+        "es": 0.94,
+        "nl": 0.996
+      },
+      "sae_top_5_test_accuracy": {
+        "en": 0.998,
+        "fr": 0.99,
+        "de": 0.999,
+        "es": 0.995,
+        "nl": 1.0
+      }
+    }
+  }
+}

old_relu_eval_results/sparse_probing/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_2_custom_sae_eval_results.json

@@ -0,0 +1,670 @@
+{
+  "eval_type_id": "sparse_probing",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "LabHC/bias_in_bios_class_set2",
+      "LabHC/bias_in_bios_class_set3",
+      "canrager/amazon_reviews_mcauley_1and5",
+      "canrager/amazon_reviews_mcauley_1and5_sentiment",
+      "codeparrot/github-code",
+      "fancyzhx/ag_news",
+      "Helsinki-NLP/europarl"
+    ],
+    "probe_train_set_size": 4000,
+    "probe_test_set_size": 1000,
+    "context_length": 128,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "model_name": "gemma-2-2b",
+    "k_values": [
+      1,
+      2,
+      5
+    ],
+    "lower_vram_usage": false
+  },
+  "eval_id": "b46e4809-b52d-4af3-a3b9-c16c6abcee8d",
+  "datetime_epoch_millis": 1738810625768,
+  "eval_result_metrics": {
+    "llm": {
+      "llm_test_accuracy": 0.9598687920719384,
+      "llm_top_1_test_accuracy": 0.6589812500000001,
+      "llm_top_2_test_accuracy": 0.7174812500000001,
+      "llm_top_5_test_accuracy": 0.7826062500000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null
+    },
+    "sae": {
+      "sae_test_accuracy": 0.9566000409424305,
+      "sae_top_1_test_accuracy": 0.8076125000000001,
+      "sae_top_2_test_accuracy": 0.85893125,
+      "sae_top_5_test_accuracy": 0.8915875,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_results",
+      "llm_test_accuracy": 0.9694000363349915,
+      "llm_top_1_test_accuracy": 0.6436000000000001,
+      "llm_top_2_test_accuracy": 0.6874,
+      "llm_top_5_test_accuracy": 0.7908,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9660000443458557,
+      "sae_top_1_test_accuracy": 0.7590000000000001,
+      "sae_top_2_test_accuracy": 0.8640000000000001,
+      "sae_top_5_test_accuracy": 0.8916000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set2_results",
+      "llm_test_accuracy": 0.9544000387191772,
+      "llm_top_1_test_accuracy": 0.67,
+      "llm_top_2_test_accuracy": 0.7148,
+      "llm_top_5_test_accuracy": 0.7716,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9500000596046447,
+      "sae_top_1_test_accuracy": 0.7979999999999999,
+      "sae_top_2_test_accuracy": 0.8022,
+      "sae_top_5_test_accuracy": 0.8744,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set3_results",
+      "llm_test_accuracy": 0.9320000410079956,
+      "llm_top_1_test_accuracy": 0.6918,
+      "llm_top_2_test_accuracy": 0.7338,
+      "llm_top_5_test_accuracy": 0.765,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9332000494003296,
+      "sae_top_1_test_accuracy": 0.8299999999999998,
+      "sae_top_2_test_accuracy": 0.8457999999999999,
+      "sae_top_5_test_accuracy": 0.8716000000000002,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_results",
+      "llm_test_accuracy": 0.9196000456809997,
+      "llm_top_1_test_accuracy": 0.6048,
+      "llm_top_2_test_accuracy": 0.6406000000000001,
+      "llm_top_5_test_accuracy": 0.6696,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9124000430107116,
+      "sae_top_1_test_accuracy": 0.7344000000000002,
+      "sae_top_2_test_accuracy": 0.7668,
+      "sae_top_5_test_accuracy": 0.8311999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_sentiment_results",
+      "llm_test_accuracy": 0.9820000529289246,
+      "llm_top_1_test_accuracy": 0.672,
+      "llm_top_2_test_accuracy": 0.724,
+      "llm_top_5_test_accuracy": 0.766,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9745000302791595,
+      "sae_top_1_test_accuracy": 0.858,
+      "sae_top_2_test_accuracy": 0.932,
+      "sae_top_5_test_accuracy": 0.932,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "codeparrot/github-code_results",
+      "llm_test_accuracy": 0.969200050830841,
+      "llm_top_1_test_accuracy": 0.6428,
+      "llm_top_2_test_accuracy": 0.6920000000000001,
+      "llm_top_5_test_accuracy": 0.7656000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.963800048828125,
+      "sae_top_1_test_accuracy": 0.8106000000000002,
+      "sae_top_2_test_accuracy": 0.8326,
+      "sae_top_5_test_accuracy": 0.8598000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "fancyzhx/ag_news_results",
+      "llm_test_accuracy": 0.9527500420808792,
+      "llm_top_1_test_accuracy": 0.69725,
+      "llm_top_2_test_accuracy": 0.76625,
+      "llm_top_5_test_accuracy": 0.8192499999999999,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9535000324249268,
+      "sae_top_1_test_accuracy": 0.7625000000000001,
+      "sae_top_2_test_accuracy": 0.8552500000000001,
+      "sae_top_5_test_accuracy": 0.8755000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "Helsinki-NLP/europarl_results",
+      "llm_test_accuracy": 0.9996000289916992,
+      "llm_top_1_test_accuracy": 0.6496000000000001,
+      "llm_top_2_test_accuracy": 0.7809999999999999,
+      "llm_top_5_test_accuracy": 0.913,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9994000196456909,
+      "sae_top_1_test_accuracy": 0.9084,
+      "sae_top_2_test_accuracy": 0.9728,
+      "sae_top_5_test_accuracy": 0.9966000000000002,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_2",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1_results": {
+      "sae_test_accuracy": {
+        "0": 0.9510000348091125,
+        "1": 0.9630000591278076,
+        "2": 0.9520000219345093,
+        "6": 0.9910000562667847,
+        "9": 0.9730000495910645
+      },
+      "llm_test_accuracy": {
+        "0": 0.9510000348091125,
+        "1": 0.9670000672340393,
+        "2": 0.9520000219345093,
+        "6": 0.9930000305175781,
+        "9": 0.984000027179718
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.568,
+        "1": 0.629,
+        "2": 0.679,
+        "6": 0.791,
+        "9": 0.551
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.585,
+        "1": 0.666,
+        "2": 0.673,
+        "6": 0.801,
+        "9": 0.712
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.72,
+        "1": 0.707,
+        "2": 0.764,
+        "6": 0.899,
+        "9": 0.864
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.617,
+        "1": 0.643,
+        "2": 0.852,
+        "6": 0.757,
+        "9": 0.926
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.855,
+        "1": 0.666,
+        "2": 0.888,
+        "6": 0.98,
+        "9": 0.931
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.886,
+        "1": 0.751,
+        "2": 0.908,
+        "6": 0.986,
+        "9": 0.927
+      }
+    },
+    "LabHC/bias_in_bios_class_set2_results": {
+      "sae_test_accuracy": {
+        "11": 0.9590000510215759,
+        "13": 0.9530000686645508,
+        "14": 0.9580000638961792,
+        "18": 0.9250000715255737,
+        "19": 0.9550000429153442
+      },
+      "llm_test_accuracy": {
+        "11": 0.971000075340271,
+        "13": 0.9520000219345093,
+        "14": 0.956000030040741,
+        "18": 0.9360000491142273,
+        "19": 0.9570000171661377
+      },
+      "llm_top_1_test_accuracy": {
+        "11": 0.561,
+        "13": 0.672,
+        "14": 0.631,
+        "18": 0.7,
+        "19": 0.786
+      },
+      "llm_top_2_test_accuracy": {
+        "11": 0.69,
+        "13": 0.72,
+        "14": 0.677,
+        "18": 0.721,
+        "19": 0.766
+      },
+      "llm_top_5_test_accuracy": {
+        "11": 0.776,
+        "13": 0.742,
+        "14": 0.768,
+        "18": 0.731,
+        "19": 0.841
+      },
+      "sae_top_1_test_accuracy": {
+        "11": 0.854,
+        "13": 0.707,
+        "14": 0.864,
+        "18": 0.722,
+        "19": 0.843
+      },
+      "sae_top_2_test_accuracy": {
+        "11": 0.855,
+        "13": 0.686,
+        "14": 0.882,
+        "18": 0.725,
+        "19": 0.863
+      },
+      "sae_top_5_test_accuracy": {
+        "11": 0.954,
+        "13": 0.792,
+        "14": 0.882,
+        "18": 0.898,
+        "19": 0.846
+      }
+    },
+    "LabHC/bias_in_bios_class_set3_results": {
+      "sae_test_accuracy": {
+        "20": 0.9650000333786011,
+        "21": 0.9260000586509705,
+        "22": 0.909000039100647,
+        "25": 0.971000075340271,
+        "26": 0.8950000405311584
+      },
+      "llm_test_accuracy": {
+        "20": 0.956000030040741,
+        "21": 0.9260000586509705,
+        "22": 0.9170000553131104,
+        "25": 0.9640000462532043,
+        "26": 0.8970000147819519
+      },
+      "llm_top_1_test_accuracy": {
+        "20": 0.699,
+        "21": 0.772,
+        "22": 0.641,
+        "25": 0.703,
+        "26": 0.644
+      },
+      "llm_top_2_test_accuracy": {
+        "20": 0.811,
+        "21": 0.757,
+        "22": 0.655,
+        "25": 0.762,
+        "26": 0.684
+      },
+      "llm_top_5_test_accuracy": {
+        "20": 0.861,
+        "21": 0.788,
+        "22": 0.712,
+        "25": 0.796,
+        "26": 0.668
+      },
+      "sae_top_1_test_accuracy": {
+        "20": 0.876,
+        "21": 0.789,
+        "22": 0.885,
+        "25": 0.896,
+        "26": 0.704
+      },
+      "sae_top_2_test_accuracy": {
+        "20": 0.907,
+        "21": 0.799,
+        "22": 0.888,
+        "25": 0.889,
+        "26": 0.746
+      },
+      "sae_top_5_test_accuracy": {
+        "20": 0.939,
+        "21": 0.847,
+        "22": 0.901,
+        "25": 0.904,
+        "26": 0.767
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_results": {
+      "sae_test_accuracy": {
+        "1": 0.9490000605583191,
+        "2": 0.9220000505447388,
+        "3": 0.9120000600814819,
+        "5": 0.9200000166893005,
+        "6": 0.859000027179718
+      },
+      "llm_test_accuracy": {
+        "1": 0.956000030040741,
+        "2": 0.937000036239624,
+        "3": 0.9160000681877136,
+        "5": 0.9170000553131104,
+        "6": 0.8720000386238098
+      },
+      "llm_top_1_test_accuracy": {
+        "1": 0.665,
+        "2": 0.596,
+        "3": 0.599,
+        "5": 0.576,
+        "6": 0.588
+      },
+      "llm_top_2_test_accuracy": {
+        "1": 0.73,
+        "2": 0.632,
+        "3": 0.617,
+        "5": 0.615,
+        "6": 0.609
+      },
+      "llm_top_5_test_accuracy": {
+        "1": 0.758,
+        "2": 0.646,
+        "3": 0.627,
+        "5": 0.646,
+        "6": 0.671
+      },
+      "sae_top_1_test_accuracy": {
+        "1": 0.854,
+        "2": 0.779,
+        "3": 0.676,
+        "5": 0.676,
+        "6": 0.687
+      },
+      "sae_top_2_test_accuracy": {
+        "1": 0.859,
+        "2": 0.793,
+        "3": 0.695,
+        "5": 0.736,
+        "6": 0.751
+      },
+      "sae_top_5_test_accuracy": {
+        "1": 0.885,
+        "2": 0.854,
+        "3": 0.805,
+        "5": 0.859,
+        "6": 0.753
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_sentiment_results": {
+      "sae_test_accuracy": {
+        "1.0": 0.9740000367164612,
+        "5.0": 0.9750000238418579
+      },
+      "llm_test_accuracy": {
+        "1.0": 0.9820000529289246,
+        "5.0": 0.9820000529289246
+      },
+      "llm_top_1_test_accuracy": {
+        "1.0": 0.672,
+        "5.0": 0.672
+      },
+      "llm_top_2_test_accuracy": {
+        "1.0": 0.724,
+        "5.0": 0.724
+      },
+      "llm_top_5_test_accuracy": {
+        "1.0": 0.766,
+        "5.0": 0.766
+      },
+      "sae_top_1_test_accuracy": {
+        "1.0": 0.858,
+        "5.0": 0.858
+      },
+      "sae_top_2_test_accuracy": {
+        "1.0": 0.932,
+        "5.0": 0.932
+      },
+      "sae_top_5_test_accuracy": {
+        "1.0": 0.932,
+        "5.0": 0.932
+      }
+    },
+    "codeparrot/github-code_results": {
+      "sae_test_accuracy": {
+        "C": 0.9390000700950623,
+        "Python": 0.9830000400543213,
+        "HTML": 0.9860000610351562,
+        "Java": 0.9570000171661377,
+        "PHP": 0.9540000557899475
+      },
+      "llm_test_accuracy": {
+        "C": 0.9530000686645508,
+        "Python": 0.9860000610351562,
+        "HTML": 0.9880000352859497,
+        "Java": 0.9640000462532043,
+        "PHP": 0.9550000429153442
+      },
+      "llm_top_1_test_accuracy": {
+        "C": 0.657,
+        "Python": 0.637,
+        "HTML": 0.714,
+        "Java": 0.624,
+        "PHP": 0.582
+      },
+      "llm_top_2_test_accuracy": {
+        "C": 0.658,
+        "Python": 0.664,
+        "HTML": 0.801,
+        "Java": 0.697,
+        "PHP": 0.64
+      },
+      "llm_top_5_test_accuracy": {
+        "C": 0.752,
+        "Python": 0.726,
+        "HTML": 0.928,
+        "Java": 0.728,
+        "PHP": 0.694
+      },
+      "sae_top_1_test_accuracy": {
+        "C": 0.632,
+        "Python": 0.914,
+        "HTML": 0.88,
+        "Java": 0.708,
+        "PHP": 0.919
+      },
+      "sae_top_2_test_accuracy": {
+        "C": 0.639,
+        "Python": 0.923,
+        "HTML": 0.893,
+        "Java": 0.785,
+        "PHP": 0.923
+      },
+      "sae_top_5_test_accuracy": {
+        "C": 0.714,
+        "Python": 0.942,
+        "HTML": 0.927,
+        "Java": 0.793,
+        "PHP": 0.923
+      }
+    },
+    "fancyzhx/ag_news_results": {
+      "sae_test_accuracy": {
+        "0": 0.9420000314712524,
+        "1": 0.9890000224113464,
+        "2": 0.9290000200271606,
+        "3": 0.9540000557899475
+      },
+      "llm_test_accuracy": {
+        "0": 0.9380000233650208,
+        "1": 0.9880000352859497,
+        "2": 0.9300000667572021,
+        "3": 0.9550000429153442
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.805,
+        "1": 0.67,
+        "2": 0.648,
+        "3": 0.666
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.794,
+        "1": 0.795,
+        "2": 0.686,
+        "3": 0.79
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.818,
+        "1": 0.867,
+        "2": 0.756,
+        "3": 0.836
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.683,
+        "1": 0.936,
+        "2": 0.774,
+        "3": 0.657
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.848,
+        "1": 0.932,
+        "2": 0.828,
+        "3": 0.813
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.852,
+        "1": 0.946,
+        "2": 0.841,
+        "3": 0.863
+      }
+    },
+    "Helsinki-NLP/europarl_results": {
+      "sae_test_accuracy": {
+        "en": 0.9980000257492065,
+        "fr": 1.0,
+        "de": 1.0,
+        "es": 1.0,
+        "nl": 0.999000072479248
+      },
+      "llm_test_accuracy": {
+        "en": 1.0,
+        "fr": 1.0,
+        "de": 0.999000072479248,
+        "es": 1.0,
+        "nl": 0.999000072479248
+      },
+      "llm_top_1_test_accuracy": {
+        "en": 0.748,
+        "fr": 0.59,
+        "de": 0.754,
+        "es": 0.494,
+        "nl": 0.662
+      },
+      "llm_top_2_test_accuracy": {
+        "en": 0.822,
+        "fr": 0.596,
+        "de": 0.831,
+        "es": 0.91,
+        "nl": 0.746
+      },
+      "llm_top_5_test_accuracy": {
+        "en": 0.887,
+        "fr": 0.922,
+        "de": 0.909,
+        "es": 0.982,
+        "nl": 0.865
+      },
+      "sae_top_1_test_accuracy": {
+        "en": 0.69,
+        "fr": 0.993,
+        "de": 0.917,
+        "es": 0.944,
+        "nl": 0.998
+      },
+      "sae_top_2_test_accuracy": {
+        "en": 0.999,
+        "fr": 0.994,
+        "de": 0.922,
+        "es": 0.951,
+        "nl": 0.998
+      },
+      "sae_top_5_test_accuracy": {
+        "en": 0.998,
+        "fr": 0.993,
+        "de": 0.999,
+        "es": 0.995,
+        "nl": 0.998
+      }
+    }
+  }
+}

old_relu_eval_results/sparse_probing/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_3_custom_sae_eval_results.json

@@ -0,0 +1,670 @@
+{
+  "eval_type_id": "sparse_probing",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "LabHC/bias_in_bios_class_set2",
+      "LabHC/bias_in_bios_class_set3",
+      "canrager/amazon_reviews_mcauley_1and5",
+      "canrager/amazon_reviews_mcauley_1and5_sentiment",
+      "codeparrot/github-code",
+      "fancyzhx/ag_news",
+      "Helsinki-NLP/europarl"
+    ],
+    "probe_train_set_size": 4000,
+    "probe_test_set_size": 1000,
+    "context_length": 128,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "model_name": "gemma-2-2b",
+    "k_values": [
+      1,
+      2,
+      5
+    ],
+    "lower_vram_usage": false
+  },
+  "eval_id": "6425df5a-b293-4597-bb77-df77861d6457",
+  "datetime_epoch_millis": 1738810319161,
+  "eval_result_metrics": {
+    "llm": {
+      "llm_test_accuracy": 0.9598687920719384,
+      "llm_top_1_test_accuracy": 0.6589812500000001,
+      "llm_top_2_test_accuracy": 0.7174812500000001,
+      "llm_top_5_test_accuracy": 0.7826062500000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null
+    },
+    "sae": {
+      "sae_test_accuracy": 0.9553375504910946,
+      "sae_top_1_test_accuracy": 0.77961875,
+      "sae_top_2_test_accuracy": 0.853675,
+      "sae_top_5_test_accuracy": 0.89161875,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_results",
+      "llm_test_accuracy": 0.9694000363349915,
+      "llm_top_1_test_accuracy": 0.6436000000000001,
+      "llm_top_2_test_accuracy": 0.6874,
+      "llm_top_5_test_accuracy": 0.7908,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9662000417709351,
+      "sae_top_1_test_accuracy": 0.8164,
+      "sae_top_2_test_accuracy": 0.9014,
+      "sae_top_5_test_accuracy": 0.9178,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set2_results",
+      "llm_test_accuracy": 0.9544000387191772,
+      "llm_top_1_test_accuracy": 0.67,
+      "llm_top_2_test_accuracy": 0.7148,
+      "llm_top_5_test_accuracy": 0.7716,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9474000453948974,
+      "sae_top_1_test_accuracy": 0.766,
+      "sae_top_2_test_accuracy": 0.7964,
+      "sae_top_5_test_accuracy": 0.8795999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set3_results",
+      "llm_test_accuracy": 0.9320000410079956,
+      "llm_top_1_test_accuracy": 0.6918,
+      "llm_top_2_test_accuracy": 0.7338,
+      "llm_top_5_test_accuracy": 0.765,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9244000434875488,
+      "sae_top_1_test_accuracy": 0.8320000000000001,
+      "sae_top_2_test_accuracy": 0.8568000000000001,
+      "sae_top_5_test_accuracy": 0.8774000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_results",
+      "llm_test_accuracy": 0.9196000456809997,
+      "llm_top_1_test_accuracy": 0.6048,
+      "llm_top_2_test_accuracy": 0.6406000000000001,
+      "llm_top_5_test_accuracy": 0.6696,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9170000433921814,
+      "sae_top_1_test_accuracy": 0.6744,
+      "sae_top_2_test_accuracy": 0.7302000000000001,
+      "sae_top_5_test_accuracy": 0.8102,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_sentiment_results",
+      "llm_test_accuracy": 0.9820000529289246,
+      "llm_top_1_test_accuracy": 0.672,
+      "llm_top_2_test_accuracy": 0.724,
+      "llm_top_5_test_accuracy": 0.766,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.971000075340271,
+      "sae_top_1_test_accuracy": 0.805,
+      "sae_top_2_test_accuracy": 0.934,
+      "sae_top_5_test_accuracy": 0.943,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "codeparrot/github-code_results",
+      "llm_test_accuracy": 0.969200050830841,
+      "llm_top_1_test_accuracy": 0.6428,
+      "llm_top_2_test_accuracy": 0.6920000000000001,
+      "llm_top_5_test_accuracy": 0.7656000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9680000424385071,
+      "sae_top_1_test_accuracy": 0.7978000000000001,
+      "sae_top_2_test_accuracy": 0.8178000000000001,
+      "sae_top_5_test_accuracy": 0.8513999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "fancyzhx/ag_news_results",
+      "llm_test_accuracy": 0.9527500420808792,
+      "llm_top_1_test_accuracy": 0.69725,
+      "llm_top_2_test_accuracy": 0.76625,
+      "llm_top_5_test_accuracy": 0.8192499999999999,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9495000541210175,
+      "sae_top_1_test_accuracy": 0.6577500000000001,
+      "sae_top_2_test_accuracy": 0.813,
+      "sae_top_5_test_accuracy": 0.86075,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "Helsinki-NLP/europarl_results",
+      "llm_test_accuracy": 0.9996000289916992,
+      "llm_top_1_test_accuracy": 0.6496000000000001,
+      "llm_top_2_test_accuracy": 0.7809999999999999,
+      "llm_top_5_test_accuracy": 0.913,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9992000579833984,
+      "sae_top_1_test_accuracy": 0.8876,
+      "sae_top_2_test_accuracy": 0.9798,
+      "sae_top_5_test_accuracy": 0.9928000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_3",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1_results": {
+      "sae_test_accuracy": {
+        "0": 0.9500000476837158,
+        "1": 0.9640000462532043,
+        "2": 0.9520000219345093,
+        "6": 0.9880000352859497,
+        "9": 0.9770000576972961
+      },
+      "llm_test_accuracy": {
+        "0": 0.9510000348091125,
+        "1": 0.9670000672340393,
+        "2": 0.9520000219345093,
+        "6": 0.9930000305175781,
+        "9": 0.984000027179718
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.568,
+        "1": 0.629,
+        "2": 0.679,
+        "6": 0.791,
+        "9": 0.551
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.585,
+        "1": 0.666,
+        "2": 0.673,
+        "6": 0.801,
+        "9": 0.712
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.72,
+        "1": 0.707,
+        "2": 0.764,
+        "6": 0.899,
+        "9": 0.864
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.617,
+        "1": 0.823,
+        "2": 0.843,
+        "6": 0.977,
+        "9": 0.822
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.881,
+        "1": 0.848,
+        "2": 0.884,
+        "6": 0.976,
+        "9": 0.918
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.896,
+        "1": 0.883,
+        "2": 0.904,
+        "6": 0.986,
+        "9": 0.92
+      }
+    },
+    "LabHC/bias_in_bios_class_set2_results": {
+      "sae_test_accuracy": {
+        "11": 0.9580000638961792,
+        "13": 0.956000030040741,
+        "14": 0.9550000429153442,
+        "18": 0.9110000729560852,
+        "19": 0.9570000171661377
+      },
+      "llm_test_accuracy": {
+        "11": 0.971000075340271,
+        "13": 0.9520000219345093,
+        "14": 0.956000030040741,
+        "18": 0.9360000491142273,
+        "19": 0.9570000171661377
+      },
+      "llm_top_1_test_accuracy": {
+        "11": 0.561,
+        "13": 0.672,
+        "14": 0.631,
+        "18": 0.7,
+        "19": 0.786
+      },
+      "llm_top_2_test_accuracy": {
+        "11": 0.69,
+        "13": 0.72,
+        "14": 0.677,
+        "18": 0.721,
+        "19": 0.766
+      },
+      "llm_top_5_test_accuracy": {
+        "11": 0.776,
+        "13": 0.742,
+        "14": 0.768,
+        "18": 0.731,
+        "19": 0.841
+      },
+      "sae_top_1_test_accuracy": {
+        "11": 0.733,
+        "13": 0.69,
+        "14": 0.839,
+        "18": 0.735,
+        "19": 0.833
+      },
+      "sae_top_2_test_accuracy": {
+        "11": 0.859,
+        "13": 0.705,
+        "14": 0.85,
+        "18": 0.728,
+        "19": 0.84
+      },
+      "sae_top_5_test_accuracy": {
+        "11": 0.947,
+        "13": 0.79,
+        "14": 0.885,
+        "18": 0.917,
+        "19": 0.859
+      }
+    },
+    "LabHC/bias_in_bios_class_set3_results": {
+      "sae_test_accuracy": {
+        "20": 0.9550000429153442,
+        "21": 0.9220000505447388,
+        "22": 0.9100000262260437,
+        "25": 0.9540000557899475,
+        "26": 0.8810000419616699
+      },
+      "llm_test_accuracy": {
+        "20": 0.956000030040741,
+        "21": 0.9260000586509705,
+        "22": 0.9170000553131104,
+        "25": 0.9640000462532043,
+        "26": 0.8970000147819519
+      },
+      "llm_top_1_test_accuracy": {
+        "20": 0.699,
+        "21": 0.772,
+        "22": 0.641,
+        "25": 0.703,
+        "26": 0.644
+      },
+      "llm_top_2_test_accuracy": {
+        "20": 0.811,
+        "21": 0.757,
+        "22": 0.655,
+        "25": 0.762,
+        "26": 0.684
+      },
+      "llm_top_5_test_accuracy": {
+        "20": 0.861,
+        "21": 0.788,
+        "22": 0.712,
+        "25": 0.796,
+        "26": 0.668
+      },
+      "sae_top_1_test_accuracy": {
+        "20": 0.882,
+        "21": 0.811,
+        "22": 0.888,
+        "25": 0.879,
+        "26": 0.7
+      },
+      "sae_top_2_test_accuracy": {
+        "20": 0.899,
+        "21": 0.834,
+        "22": 0.899,
+        "25": 0.882,
+        "26": 0.77
+      },
+      "sae_top_5_test_accuracy": {
+        "20": 0.937,
+        "21": 0.855,
+        "22": 0.885,
+        "25": 0.902,
+        "26": 0.808
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_results": {
+      "sae_test_accuracy": {
+        "1": 0.9420000314712524,
+        "2": 0.9360000491142273,
+        "3": 0.9260000586509705,
+        "5": 0.9170000553131104,
+        "6": 0.8640000224113464
+      },
+      "llm_test_accuracy": {
+        "1": 0.956000030040741,
+        "2": 0.937000036239624,
+        "3": 0.9160000681877136,
+        "5": 0.9170000553131104,
+        "6": 0.8720000386238098
+      },
+      "llm_top_1_test_accuracy": {
+        "1": 0.665,
+        "2": 0.596,
+        "3": 0.599,
+        "5": 0.576,
+        "6": 0.588
+      },
+      "llm_top_2_test_accuracy": {
+        "1": 0.73,
+        "2": 0.632,
+        "3": 0.617,
+        "5": 0.615,
+        "6": 0.609
+      },
+      "llm_top_5_test_accuracy": {
+        "1": 0.758,
+        "2": 0.646,
+        "3": 0.627,
+        "5": 0.646,
+        "6": 0.671
+      },
+      "sae_top_1_test_accuracy": {
+        "1": 0.861,
+        "2": 0.629,
+        "3": 0.597,
+        "5": 0.605,
+        "6": 0.68
+      },
+      "sae_top_2_test_accuracy": {
+        "1": 0.903,
+        "2": 0.752,
+        "3": 0.648,
+        "5": 0.631,
+        "6": 0.717
+      },
+      "sae_top_5_test_accuracy": {
+        "1": 0.918,
+        "2": 0.809,
+        "3": 0.698,
+        "5": 0.873,
+        "6": 0.753
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_sentiment_results": {
+      "sae_test_accuracy": {
+        "1.0": 0.971000075340271,
+        "5.0": 0.971000075340271
+      },
+      "llm_test_accuracy": {
+        "1.0": 0.9820000529289246,
+        "5.0": 0.9820000529289246
+      },
+      "llm_top_1_test_accuracy": {
+        "1.0": 0.672,
+        "5.0": 0.672
+      },
+      "llm_top_2_test_accuracy": {
+        "1.0": 0.724,
+        "5.0": 0.724
+      },
+      "llm_top_5_test_accuracy": {
+        "1.0": 0.766,
+        "5.0": 0.766
+      },
+      "sae_top_1_test_accuracy": {
+        "1.0": 0.805,
+        "5.0": 0.805
+      },
+      "sae_top_2_test_accuracy": {
+        "1.0": 0.934,
+        "5.0": 0.934
+      },
+      "sae_top_5_test_accuracy": {
+        "1.0": 0.943,
+        "5.0": 0.943
+      }
+    },
+    "codeparrot/github-code_results": {
+      "sae_test_accuracy": {
+        "C": 0.9620000720024109,
+        "Python": 0.984000027179718,
+        "HTML": 0.984000027179718,
+        "Java": 0.9550000429153442,
+        "PHP": 0.9550000429153442
+      },
+      "llm_test_accuracy": {
+        "C": 0.9530000686645508,
+        "Python": 0.9860000610351562,
+        "HTML": 0.9880000352859497,
+        "Java": 0.9640000462532043,
+        "PHP": 0.9550000429153442
+      },
+      "llm_top_1_test_accuracy": {
+        "C": 0.657,
+        "Python": 0.637,
+        "HTML": 0.714,
+        "Java": 0.624,
+        "PHP": 0.582
+      },
+      "llm_top_2_test_accuracy": {
+        "C": 0.658,
+        "Python": 0.664,
+        "HTML": 0.801,
+        "Java": 0.697,
+        "PHP": 0.64
+      },
+      "llm_top_5_test_accuracy": {
+        "C": 0.752,
+        "Python": 0.726,
+        "HTML": 0.928,
+        "Java": 0.728,
+        "PHP": 0.694
+      },
+      "sae_top_1_test_accuracy": {
+        "C": 0.604,
+        "Python": 0.906,
+        "HTML": 0.876,
+        "Java": 0.713,
+        "PHP": 0.89
+      },
+      "sae_top_2_test_accuracy": {
+        "C": 0.69,
+        "Python": 0.914,
+        "HTML": 0.889,
+        "Java": 0.699,
+        "PHP": 0.897
+      },
+      "sae_top_5_test_accuracy": {
+        "C": 0.717,
+        "Python": 0.92,
+        "HTML": 0.915,
+        "Java": 0.79,
+        "PHP": 0.915
+      }
+    },
+    "fancyzhx/ag_news_results": {
+      "sae_test_accuracy": {
+        "0": 0.9350000619888306,
+        "1": 0.9860000610351562,
+        "2": 0.9310000538825989,
+        "3": 0.9460000395774841
+      },
+      "llm_test_accuracy": {
+        "0": 0.9380000233650208,
+        "1": 0.9880000352859497,
+        "2": 0.9300000667572021,
+        "3": 0.9550000429153442
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.805,
+        "1": 0.67,
+        "2": 0.648,
+        "3": 0.666
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.794,
+        "1": 0.795,
+        "2": 0.686,
+        "3": 0.79
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.818,
+        "1": 0.867,
+        "2": 0.756,
+        "3": 0.836
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.715,
+        "1": 0.606,
+        "2": 0.695,
+        "3": 0.615
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.839,
+        "1": 0.837,
+        "2": 0.815,
+        "3": 0.761
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.871,
+        "1": 0.935,
+        "2": 0.818,
+        "3": 0.819
+      }
+    },
+    "Helsinki-NLP/europarl_results": {
+      "sae_test_accuracy": {
+        "en": 0.999000072479248,
+        "fr": 0.999000072479248,
+        "de": 1.0,
+        "es": 0.999000072479248,
+        "nl": 0.999000072479248
+      },
+      "llm_test_accuracy": {
+        "en": 1.0,
+        "fr": 1.0,
+        "de": 0.999000072479248,
+        "es": 1.0,
+        "nl": 0.999000072479248
+      },
+      "llm_top_1_test_accuracy": {
+        "en": 0.748,
+        "fr": 0.59,
+        "de": 0.754,
+        "es": 0.494,
+        "nl": 0.662
+      },
+      "llm_top_2_test_accuracy": {
+        "en": 0.822,
+        "fr": 0.596,
+        "de": 0.831,
+        "es": 0.91,
+        "nl": 0.746
+      },
+      "llm_top_5_test_accuracy": {
+        "en": 0.887,
+        "fr": 0.922,
+        "de": 0.909,
+        "es": 0.982,
+        "nl": 0.865
+      },
+      "sae_top_1_test_accuracy": {
+        "en": 0.685,
+        "fr": 0.984,
+        "de": 0.918,
+        "es": 0.942,
+        "nl": 0.909
+      },
+      "sae_top_2_test_accuracy": {
+        "en": 0.996,
+        "fr": 0.99,
+        "de": 0.926,
+        "es": 0.991,
+        "nl": 0.996
+      },
+      "sae_top_5_test_accuracy": {
+        "en": 0.999,
+        "fr": 0.991,
+        "de": 0.986,
+        "es": 0.99,
+        "nl": 0.998
+      }
+    }
+  }
+}

old_relu_eval_results/sparse_probing/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_4_custom_sae_eval_results.json

@@ -0,0 +1,670 @@
+{
+  "eval_type_id": "sparse_probing",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "LabHC/bias_in_bios_class_set2",
+      "LabHC/bias_in_bios_class_set3",
+      "canrager/amazon_reviews_mcauley_1and5",
+      "canrager/amazon_reviews_mcauley_1and5_sentiment",
+      "codeparrot/github-code",
+      "fancyzhx/ag_news",
+      "Helsinki-NLP/europarl"
+    ],
+    "probe_train_set_size": 4000,
+    "probe_test_set_size": 1000,
+    "context_length": 128,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "model_name": "gemma-2-2b",
+    "k_values": [
+      1,
+      2,
+      5
+    ],
+    "lower_vram_usage": false
+  },
+  "eval_id": "79d387eb-180f-4d7c-8356-24c5e2fb2eca",
+  "datetime_epoch_millis": 1738810868662,
+  "eval_result_metrics": {
+    "llm": {
+      "llm_test_accuracy": 0.9598687920719384,
+      "llm_top_1_test_accuracy": 0.6589812500000001,
+      "llm_top_2_test_accuracy": 0.7174812500000001,
+      "llm_top_5_test_accuracy": 0.7826062500000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null
+    },
+    "sae": {
+      "sae_test_accuracy": 0.9545250445604324,
+      "sae_top_1_test_accuracy": 0.7856687499999999,
+      "sae_top_2_test_accuracy": 0.8240375,
+      "sae_top_5_test_accuracy": 0.8696375000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_results",
+      "llm_test_accuracy": 0.9694000363349915,
+      "llm_top_1_test_accuracy": 0.6436000000000001,
+      "llm_top_2_test_accuracy": 0.6874,
+      "llm_top_5_test_accuracy": 0.7908,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9622000455856323,
+      "sae_top_1_test_accuracy": 0.7978000000000001,
+      "sae_top_2_test_accuracy": 0.8472,
+      "sae_top_5_test_accuracy": 0.8636000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set2_results",
+      "llm_test_accuracy": 0.9544000387191772,
+      "llm_top_1_test_accuracy": 0.67,
+      "llm_top_2_test_accuracy": 0.7148,
+      "llm_top_5_test_accuracy": 0.7716,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9474000453948974,
+      "sae_top_1_test_accuracy": 0.7654,
+      "sae_top_2_test_accuracy": 0.7933999999999999,
+      "sae_top_5_test_accuracy": 0.8804000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set3_results",
+      "llm_test_accuracy": 0.9320000410079956,
+      "llm_top_1_test_accuracy": 0.6918,
+      "llm_top_2_test_accuracy": 0.7338,
+      "llm_top_5_test_accuracy": 0.765,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9278000354766845,
+      "sae_top_1_test_accuracy": 0.8138,
+      "sae_top_2_test_accuracy": 0.8508000000000001,
+      "sae_top_5_test_accuracy": 0.8704000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_results",
+      "llm_test_accuracy": 0.9196000456809997,
+      "llm_top_1_test_accuracy": 0.6048,
+      "llm_top_2_test_accuracy": 0.6406000000000001,
+      "llm_top_5_test_accuracy": 0.6696,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9180000424385071,
+      "sae_top_1_test_accuracy": 0.6996,
+      "sae_top_2_test_accuracy": 0.7652,
+      "sae_top_5_test_accuracy": 0.8160000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_sentiment_results",
+      "llm_test_accuracy": 0.9820000529289246,
+      "llm_top_1_test_accuracy": 0.672,
+      "llm_top_2_test_accuracy": 0.724,
+      "llm_top_5_test_accuracy": 0.766,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9690000414848328,
+      "sae_top_1_test_accuracy": 0.8,
+      "sae_top_2_test_accuracy": 0.813,
+      "sae_top_5_test_accuracy": 0.837,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "codeparrot/github-code_results",
+      "llm_test_accuracy": 0.969200050830841,
+      "llm_top_1_test_accuracy": 0.6428,
+      "llm_top_2_test_accuracy": 0.6920000000000001,
+      "llm_top_5_test_accuracy": 0.7656000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9656000375747681,
+      "sae_top_1_test_accuracy": 0.7926,
+      "sae_top_2_test_accuracy": 0.8016,
+      "sae_top_5_test_accuracy": 0.8466000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "fancyzhx/ag_news_results",
+      "llm_test_accuracy": 0.9527500420808792,
+      "llm_top_1_test_accuracy": 0.69725,
+      "llm_top_2_test_accuracy": 0.76625,
+      "llm_top_5_test_accuracy": 0.8192499999999999,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9490000605583191,
+      "sae_top_1_test_accuracy": 0.74175,
+      "sae_top_2_test_accuracy": 0.7815,
+      "sae_top_5_test_accuracy": 0.8514999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "Helsinki-NLP/europarl_results",
+      "llm_test_accuracy": 0.9996000289916992,
+      "llm_top_1_test_accuracy": 0.6496000000000001,
+      "llm_top_2_test_accuracy": 0.7809999999999999,
+      "llm_top_5_test_accuracy": 0.913,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9972000479698181,
+      "sae_top_1_test_accuracy": 0.8744,
+      "sae_top_2_test_accuracy": 0.9395999999999999,
+      "sae_top_5_test_accuracy": 0.9916,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_4",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1_results": {
+      "sae_test_accuracy": {
+        "0": 0.9390000700950623,
+        "1": 0.9650000333786011,
+        "2": 0.9470000267028809,
+        "6": 0.9820000529289246,
+        "9": 0.9780000448226929
+      },
+      "llm_test_accuracy": {
+        "0": 0.9510000348091125,
+        "1": 0.9670000672340393,
+        "2": 0.9520000219345093,
+        "6": 0.9930000305175781,
+        "9": 0.984000027179718
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.568,
+        "1": 0.629,
+        "2": 0.679,
+        "6": 0.791,
+        "9": 0.551
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.585,
+        "1": 0.666,
+        "2": 0.673,
+        "6": 0.801,
+        "9": 0.712
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.72,
+        "1": 0.707,
+        "2": 0.764,
+        "6": 0.899,
+        "9": 0.864
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.621,
+        "1": 0.644,
+        "2": 0.856,
+        "6": 0.978,
+        "9": 0.89
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.653,
+        "1": 0.817,
+        "2": 0.878,
+        "6": 0.976,
+        "9": 0.912
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.676,
+        "1": 0.864,
+        "2": 0.889,
+        "6": 0.987,
+        "9": 0.902
+      }
+    },
+    "LabHC/bias_in_bios_class_set2_results": {
+      "sae_test_accuracy": {
+        "11": 0.9570000171661377,
+        "13": 0.9490000605583191,
+        "14": 0.9550000429153442,
+        "18": 0.9180000424385071,
+        "19": 0.9580000638961792
+      },
+      "llm_test_accuracy": {
+        "11": 0.971000075340271,
+        "13": 0.9520000219345093,
+        "14": 0.956000030040741,
+        "18": 0.9360000491142273,
+        "19": 0.9570000171661377
+      },
+      "llm_top_1_test_accuracy": {
+        "11": 0.561,
+        "13": 0.672,
+        "14": 0.631,
+        "18": 0.7,
+        "19": 0.786
+      },
+      "llm_top_2_test_accuracy": {
+        "11": 0.69,
+        "13": 0.72,
+        "14": 0.677,
+        "18": 0.721,
+        "19": 0.766
+      },
+      "llm_top_5_test_accuracy": {
+        "11": 0.776,
+        "13": 0.742,
+        "14": 0.768,
+        "18": 0.731,
+        "19": 0.841
+      },
+      "sae_top_1_test_accuracy": {
+        "11": 0.721,
+        "13": 0.695,
+        "14": 0.859,
+        "18": 0.725,
+        "19": 0.827
+      },
+      "sae_top_2_test_accuracy": {
+        "11": 0.859,
+        "13": 0.685,
+        "14": 0.868,
+        "18": 0.731,
+        "19": 0.824
+      },
+      "sae_top_5_test_accuracy": {
+        "11": 0.927,
+        "13": 0.845,
+        "14": 0.885,
+        "18": 0.903,
+        "19": 0.842
+      }
+    },
+    "LabHC/bias_in_bios_class_set3_results": {
+      "sae_test_accuracy": {
+        "20": 0.9540000557899475,
+        "21": 0.9200000166893005,
+        "22": 0.9140000343322754,
+        "25": 0.9610000252723694,
+        "26": 0.89000004529953
+      },
+      "llm_test_accuracy": {
+        "20": 0.956000030040741,
+        "21": 0.9260000586509705,
+        "22": 0.9170000553131104,
+        "25": 0.9640000462532043,
+        "26": 0.8970000147819519
+      },
+      "llm_top_1_test_accuracy": {
+        "20": 0.699,
+        "21": 0.772,
+        "22": 0.641,
+        "25": 0.703,
+        "26": 0.644
+      },
+      "llm_top_2_test_accuracy": {
+        "20": 0.811,
+        "21": 0.757,
+        "22": 0.655,
+        "25": 0.762,
+        "26": 0.684
+      },
+      "llm_top_5_test_accuracy": {
+        "20": 0.861,
+        "21": 0.788,
+        "22": 0.712,
+        "25": 0.796,
+        "26": 0.668
+      },
+      "sae_top_1_test_accuracy": {
+        "20": 0.866,
+        "21": 0.729,
+        "22": 0.883,
+        "25": 0.887,
+        "26": 0.704
+      },
+      "sae_top_2_test_accuracy": {
+        "20": 0.895,
+        "21": 0.821,
+        "22": 0.876,
+        "25": 0.893,
+        "26": 0.769
+      },
+      "sae_top_5_test_accuracy": {
+        "20": 0.932,
+        "21": 0.851,
+        "22": 0.885,
+        "25": 0.905,
+        "26": 0.779
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_results": {
+      "sae_test_accuracy": {
+        "1": 0.9430000185966492,
+        "2": 0.9290000200271606,
+        "3": 0.921000063419342,
+        "5": 0.9260000586509705,
+        "6": 0.8710000514984131
+      },
+      "llm_test_accuracy": {
+        "1": 0.956000030040741,
+        "2": 0.937000036239624,
+        "3": 0.9160000681877136,
+        "5": 0.9170000553131104,
+        "6": 0.8720000386238098
+      },
+      "llm_top_1_test_accuracy": {
+        "1": 0.665,
+        "2": 0.596,
+        "3": 0.599,
+        "5": 0.576,
+        "6": 0.588
+      },
+      "llm_top_2_test_accuracy": {
+        "1": 0.73,
+        "2": 0.632,
+        "3": 0.617,
+        "5": 0.615,
+        "6": 0.609
+      },
+      "llm_top_5_test_accuracy": {
+        "1": 0.758,
+        "2": 0.646,
+        "3": 0.627,
+        "5": 0.646,
+        "6": 0.671
+      },
+      "sae_top_1_test_accuracy": {
+        "1": 0.893,
+        "2": 0.596,
+        "3": 0.569,
+        "5": 0.771,
+        "6": 0.669
+      },
+      "sae_top_2_test_accuracy": {
+        "1": 0.892,
+        "2": 0.772,
+        "3": 0.669,
+        "5": 0.774,
+        "6": 0.719
+      },
+      "sae_top_5_test_accuracy": {
+        "1": 0.888,
+        "2": 0.864,
+        "3": 0.711,
+        "5": 0.857,
+        "6": 0.76
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_sentiment_results": {
+      "sae_test_accuracy": {
+        "1.0": 0.968000054359436,
+        "5.0": 0.9700000286102295
+      },
+      "llm_test_accuracy": {
+        "1.0": 0.9820000529289246,
+        "5.0": 0.9820000529289246
+      },
+      "llm_top_1_test_accuracy": {
+        "1.0": 0.672,
+        "5.0": 0.672
+      },
+      "llm_top_2_test_accuracy": {
+        "1.0": 0.724,
+        "5.0": 0.724
+      },
+      "llm_top_5_test_accuracy": {
+        "1.0": 0.766,
+        "5.0": 0.766
+      },
+      "sae_top_1_test_accuracy": {
+        "1.0": 0.8,
+        "5.0": 0.8
+      },
+      "sae_top_2_test_accuracy": {
+        "1.0": 0.813,
+        "5.0": 0.813
+      },
+      "sae_top_5_test_accuracy": {
+        "1.0": 0.837,
+        "5.0": 0.837
+      }
+    },
+    "codeparrot/github-code_results": {
+      "sae_test_accuracy": {
+        "C": 0.9550000429153442,
+        "Python": 0.9750000238418579,
+        "HTML": 0.9820000529289246,
+        "Java": 0.9590000510215759,
+        "PHP": 0.9570000171661377
+      },
+      "llm_test_accuracy": {
+        "C": 0.9530000686645508,
+        "Python": 0.9860000610351562,
+        "HTML": 0.9880000352859497,
+        "Java": 0.9640000462532043,
+        "PHP": 0.9550000429153442
+      },
+      "llm_top_1_test_accuracy": {
+        "C": 0.657,
+        "Python": 0.637,
+        "HTML": 0.714,
+        "Java": 0.624,
+        "PHP": 0.582
+      },
+      "llm_top_2_test_accuracy": {
+        "C": 0.658,
+        "Python": 0.664,
+        "HTML": 0.801,
+        "Java": 0.697,
+        "PHP": 0.64
+      },
+      "llm_top_5_test_accuracy": {
+        "C": 0.752,
+        "Python": 0.726,
+        "HTML": 0.928,
+        "Java": 0.728,
+        "PHP": 0.694
+      },
+      "sae_top_1_test_accuracy": {
+        "C": 0.606,
+        "Python": 0.904,
+        "HTML": 0.856,
+        "Java": 0.717,
+        "PHP": 0.88
+      },
+      "sae_top_2_test_accuracy": {
+        "C": 0.626,
+        "Python": 0.903,
+        "HTML": 0.884,
+        "Java": 0.709,
+        "PHP": 0.886
+      },
+      "sae_top_5_test_accuracy": {
+        "C": 0.742,
+        "Python": 0.925,
+        "HTML": 0.92,
+        "Java": 0.729,
+        "PHP": 0.917
+      }
+    },
+    "fancyzhx/ag_news_results": {
+      "sae_test_accuracy": {
+        "0": 0.9360000491142273,
+        "1": 0.9860000610351562,
+        "2": 0.9260000586509705,
+        "3": 0.9480000734329224
+      },
+      "llm_test_accuracy": {
+        "0": 0.9380000233650208,
+        "1": 0.9880000352859497,
+        "2": 0.9300000667572021,
+        "3": 0.9550000429153442
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.805,
+        "1": 0.67,
+        "2": 0.648,
+        "3": 0.666
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.794,
+        "1": 0.795,
+        "2": 0.686,
+        "3": 0.79
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.818,
+        "1": 0.867,
+        "2": 0.756,
+        "3": 0.836
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.832,
+        "1": 0.817,
+        "2": 0.674,
+        "3": 0.644
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.842,
+        "1": 0.821,
+        "2": 0.813,
+        "3": 0.65
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.854,
+        "1": 0.925,
+        "2": 0.814,
+        "3": 0.813
+      }
+    },
+    "Helsinki-NLP/europarl_results": {
+      "sae_test_accuracy": {
+        "en": 0.999000072479248,
+        "fr": 0.9980000257492065,
+        "de": 0.9960000514984131,
+        "es": 0.9960000514984131,
+        "nl": 0.9970000386238098
+      },
+      "llm_test_accuracy": {
+        "en": 1.0,
+        "fr": 1.0,
+        "de": 0.999000072479248,
+        "es": 1.0,
+        "nl": 0.999000072479248
+      },
+      "llm_top_1_test_accuracy": {
+        "en": 0.748,
+        "fr": 0.59,
+        "de": 0.754,
+        "es": 0.494,
+        "nl": 0.662
+      },
+      "llm_top_2_test_accuracy": {
+        "en": 0.822,
+        "fr": 0.596,
+        "de": 0.831,
+        "es": 0.91,
+        "nl": 0.746
+      },
+      "llm_top_5_test_accuracy": {
+        "en": 0.887,
+        "fr": 0.922,
+        "de": 0.909,
+        "es": 0.982,
+        "nl": 0.865
+      },
+      "sae_top_1_test_accuracy": {
+        "en": 0.696,
+        "fr": 0.986,
+        "de": 0.908,
+        "es": 0.895,
+        "nl": 0.887
+      },
+      "sae_top_2_test_accuracy": {
+        "en": 0.802,
+        "fr": 0.988,
+        "de": 0.925,
+        "es": 0.986,
+        "nl": 0.997
+      },
+      "sae_top_5_test_accuracy": {
+        "en": 1.0,
+        "fr": 0.988,
+        "de": 0.982,
+        "es": 0.991,
+        "nl": 0.997
+      }
+    }
+  }
+}

old_relu_eval_results/sparse_probing/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_5_custom_sae_eval_results.json

@@ -0,0 +1,670 @@
+{
+  "eval_type_id": "sparse_probing",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "LabHC/bias_in_bios_class_set2",
+      "LabHC/bias_in_bios_class_set3",
+      "canrager/amazon_reviews_mcauley_1and5",
+      "canrager/amazon_reviews_mcauley_1and5_sentiment",
+      "codeparrot/github-code",
+      "fancyzhx/ag_news",
+      "Helsinki-NLP/europarl"
+    ],
+    "probe_train_set_size": 4000,
+    "probe_test_set_size": 1000,
+    "context_length": 128,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "model_name": "gemma-2-2b",
+    "k_values": [
+      1,
+      2,
+      5
+    ],
+    "lower_vram_usage": false
+  },
+  "eval_id": "a24bedef-7962-4166-b149-7a1b6ad73ccc",
+  "datetime_epoch_millis": 1738810076954,
+  "eval_result_metrics": {
+    "llm": {
+      "llm_test_accuracy": 0.9598687920719384,
+      "llm_top_1_test_accuracy": 0.6589812500000001,
+      "llm_top_2_test_accuracy": 0.7174812500000001,
+      "llm_top_5_test_accuracy": 0.7826062500000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null
+    },
+    "sae": {
+      "sae_test_accuracy": 0.9497312925755977,
+      "sae_top_1_test_accuracy": 0.75526875,
+      "sae_top_2_test_accuracy": 0.8121375,
+      "sae_top_5_test_accuracy": 0.86029375,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_results",
+      "llm_test_accuracy": 0.9694000363349915,
+      "llm_top_1_test_accuracy": 0.6436000000000001,
+      "llm_top_2_test_accuracy": 0.6874,
+      "llm_top_5_test_accuracy": 0.7908,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9564000368118286,
+      "sae_top_1_test_accuracy": 0.7794,
+      "sae_top_2_test_accuracy": 0.8156000000000001,
+      "sae_top_5_test_accuracy": 0.8370000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set2_results",
+      "llm_test_accuracy": 0.9544000387191772,
+      "llm_top_1_test_accuracy": 0.67,
+      "llm_top_2_test_accuracy": 0.7148,
+      "llm_top_5_test_accuracy": 0.7716,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9434000492095947,
+      "sae_top_1_test_accuracy": 0.7693999999999999,
+      "sae_top_2_test_accuracy": 0.78,
+      "sae_top_5_test_accuracy": 0.8836,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set3_results",
+      "llm_test_accuracy": 0.9320000410079956,
+      "llm_top_1_test_accuracy": 0.6918,
+      "llm_top_2_test_accuracy": 0.7338,
+      "llm_top_5_test_accuracy": 0.765,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9236000418663025,
+      "sae_top_1_test_accuracy": 0.7532,
+      "sae_top_2_test_accuracy": 0.8549999999999999,
+      "sae_top_5_test_accuracy": 0.8664000000000002,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_results",
+      "llm_test_accuracy": 0.9196000456809997,
+      "llm_top_1_test_accuracy": 0.6048,
+      "llm_top_2_test_accuracy": 0.6406000000000001,
+      "llm_top_5_test_accuracy": 0.6696,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9022000432014465,
+      "sae_top_1_test_accuracy": 0.6662000000000001,
+      "sae_top_2_test_accuracy": 0.6942,
+      "sae_top_5_test_accuracy": 0.819,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_sentiment_results",
+      "llm_test_accuracy": 0.9820000529289246,
+      "llm_top_1_test_accuracy": 0.672,
+      "llm_top_2_test_accuracy": 0.724,
+      "llm_top_5_test_accuracy": 0.766,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9645000398159027,
+      "sae_top_1_test_accuracy": 0.786,
+      "sae_top_2_test_accuracy": 0.813,
+      "sae_top_5_test_accuracy": 0.81,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "codeparrot/github-code_results",
+      "llm_test_accuracy": 0.969200050830841,
+      "llm_top_1_test_accuracy": 0.6428,
+      "llm_top_2_test_accuracy": 0.6920000000000001,
+      "llm_top_5_test_accuracy": 0.7656000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9658000349998475,
+      "sae_top_1_test_accuracy": 0.7729999999999999,
+      "sae_top_2_test_accuracy": 0.7797999999999999,
+      "sae_top_5_test_accuracy": 0.8286,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "fancyzhx/ag_news_results",
+      "llm_test_accuracy": 0.9527500420808792,
+      "llm_top_1_test_accuracy": 0.69725,
+      "llm_top_2_test_accuracy": 0.76625,
+      "llm_top_5_test_accuracy": 0.8192499999999999,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9457500576972961,
+      "sae_top_1_test_accuracy": 0.74775,
+      "sae_top_2_test_accuracy": 0.8095,
+      "sae_top_5_test_accuracy": 0.8547499999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "Helsinki-NLP/europarl_results",
+      "llm_test_accuracy": 0.9996000289916992,
+      "llm_top_1_test_accuracy": 0.6496000000000001,
+      "llm_top_2_test_accuracy": 0.7809999999999999,
+      "llm_top_5_test_accuracy": 0.913,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9962000370025634,
+      "sae_top_1_test_accuracy": 0.7672000000000001,
+      "sae_top_2_test_accuracy": 0.95,
+      "sae_top_5_test_accuracy": 0.983,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_5",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1_results": {
+      "sae_test_accuracy": {
+        "0": 0.940000057220459,
+        "1": 0.9520000219345093,
+        "2": 0.9430000185966492,
+        "6": 0.9790000319480896,
+        "9": 0.968000054359436
+      },
+      "llm_test_accuracy": {
+        "0": 0.9510000348091125,
+        "1": 0.9670000672340393,
+        "2": 0.9520000219345093,
+        "6": 0.9930000305175781,
+        "9": 0.984000027179718
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.568,
+        "1": 0.629,
+        "2": 0.679,
+        "6": 0.791,
+        "9": 0.551
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.585,
+        "1": 0.666,
+        "2": 0.673,
+        "6": 0.801,
+        "9": 0.712
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.72,
+        "1": 0.707,
+        "2": 0.764,
+        "6": 0.899,
+        "9": 0.864
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.629,
+        "1": 0.687,
+        "2": 0.83,
+        "6": 0.978,
+        "9": 0.773
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.647,
+        "1": 0.754,
+        "2": 0.895,
+        "6": 0.98,
+        "9": 0.802
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.656,
+        "1": 0.757,
+        "2": 0.894,
+        "6": 0.983,
+        "9": 0.895
+      }
+    },
+    "LabHC/bias_in_bios_class_set2_results": {
+      "sae_test_accuracy": {
+        "11": 0.956000030040741,
+        "13": 0.9450000524520874,
+        "14": 0.9540000557899475,
+        "18": 0.9030000567436218,
+        "19": 0.9590000510215759
+      },
+      "llm_test_accuracy": {
+        "11": 0.971000075340271,
+        "13": 0.9520000219345093,
+        "14": 0.956000030040741,
+        "18": 0.9360000491142273,
+        "19": 0.9570000171661377
+      },
+      "llm_top_1_test_accuracy": {
+        "11": 0.561,
+        "13": 0.672,
+        "14": 0.631,
+        "18": 0.7,
+        "19": 0.786
+      },
+      "llm_top_2_test_accuracy": {
+        "11": 0.69,
+        "13": 0.72,
+        "14": 0.677,
+        "18": 0.721,
+        "19": 0.766
+      },
+      "llm_top_5_test_accuracy": {
+        "11": 0.776,
+        "13": 0.742,
+        "14": 0.768,
+        "18": 0.731,
+        "19": 0.841
+      },
+      "sae_top_1_test_accuracy": {
+        "11": 0.728,
+        "13": 0.705,
+        "14": 0.855,
+        "18": 0.726,
+        "19": 0.833
+      },
+      "sae_top_2_test_accuracy": {
+        "11": 0.733,
+        "13": 0.706,
+        "14": 0.866,
+        "18": 0.729,
+        "19": 0.866
+      },
+      "sae_top_5_test_accuracy": {
+        "11": 0.952,
+        "13": 0.818,
+        "14": 0.859,
+        "18": 0.909,
+        "19": 0.88
+      }
+    },
+    "LabHC/bias_in_bios_class_set3_results": {
+      "sae_test_accuracy": {
+        "20": 0.9540000557899475,
+        "21": 0.9230000376701355,
+        "22": 0.906000018119812,
+        "25": 0.956000030040741,
+        "26": 0.8790000677108765
+      },
+      "llm_test_accuracy": {
+        "20": 0.956000030040741,
+        "21": 0.9260000586509705,
+        "22": 0.9170000553131104,
+        "25": 0.9640000462532043,
+        "26": 0.8970000147819519
+      },
+      "llm_top_1_test_accuracy": {
+        "20": 0.699,
+        "21": 0.772,
+        "22": 0.641,
+        "25": 0.703,
+        "26": 0.644
+      },
+      "llm_top_2_test_accuracy": {
+        "20": 0.811,
+        "21": 0.757,
+        "22": 0.655,
+        "25": 0.762,
+        "26": 0.684
+      },
+      "llm_top_5_test_accuracy": {
+        "20": 0.861,
+        "21": 0.788,
+        "22": 0.712,
+        "25": 0.796,
+        "26": 0.668
+      },
+      "sae_top_1_test_accuracy": {
+        "20": 0.903,
+        "21": 0.751,
+        "22": 0.522,
+        "25": 0.882,
+        "26": 0.708
+      },
+      "sae_top_2_test_accuracy": {
+        "20": 0.913,
+        "21": 0.826,
+        "22": 0.869,
+        "25": 0.89,
+        "26": 0.777
+      },
+      "sae_top_5_test_accuracy": {
+        "20": 0.937,
+        "21": 0.838,
+        "22": 0.878,
+        "25": 0.909,
+        "26": 0.77
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_results": {
+      "sae_test_accuracy": {
+        "1": 0.9430000185966492,
+        "2": 0.9270000457763672,
+        "3": 0.9040000438690186,
+        "5": 0.893000066280365,
+        "6": 0.8440000414848328
+      },
+      "llm_test_accuracy": {
+        "1": 0.956000030040741,
+        "2": 0.937000036239624,
+        "3": 0.9160000681877136,
+        "5": 0.9170000553131104,
+        "6": 0.8720000386238098
+      },
+      "llm_top_1_test_accuracy": {
+        "1": 0.665,
+        "2": 0.596,
+        "3": 0.599,
+        "5": 0.576,
+        "6": 0.588
+      },
+      "llm_top_2_test_accuracy": {
+        "1": 0.73,
+        "2": 0.632,
+        "3": 0.617,
+        "5": 0.615,
+        "6": 0.609
+      },
+      "llm_top_5_test_accuracy": {
+        "1": 0.758,
+        "2": 0.646,
+        "3": 0.627,
+        "5": 0.646,
+        "6": 0.671
+      },
+      "sae_top_1_test_accuracy": {
+        "1": 0.911,
+        "2": 0.59,
+        "3": 0.543,
+        "5": 0.651,
+        "6": 0.636
+      },
+      "sae_top_2_test_accuracy": {
+        "1": 0.914,
+        "2": 0.65,
+        "3": 0.585,
+        "5": 0.645,
+        "6": 0.677
+      },
+      "sae_top_5_test_accuracy": {
+        "1": 0.936,
+        "2": 0.841,
+        "3": 0.767,
+        "5": 0.81,
+        "6": 0.741
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_sentiment_results": {
+      "sae_test_accuracy": {
+        "1.0": 0.9660000205039978,
+        "5.0": 0.9630000591278076
+      },
+      "llm_test_accuracy": {
+        "1.0": 0.9820000529289246,
+        "5.0": 0.9820000529289246
+      },
+      "llm_top_1_test_accuracy": {
+        "1.0": 0.672,
+        "5.0": 0.672
+      },
+      "llm_top_2_test_accuracy": {
+        "1.0": 0.724,
+        "5.0": 0.724
+      },
+      "llm_top_5_test_accuracy": {
+        "1.0": 0.766,
+        "5.0": 0.766
+      },
+      "sae_top_1_test_accuracy": {
+        "1.0": 0.786,
+        "5.0": 0.786
+      },
+      "sae_top_2_test_accuracy": {
+        "1.0": 0.813,
+        "5.0": 0.813
+      },
+      "sae_top_5_test_accuracy": {
+        "1.0": 0.81,
+        "5.0": 0.81
+      }
+    },
+    "codeparrot/github-code_results": {
+      "sae_test_accuracy": {
+        "C": 0.9510000348091125,
+        "Python": 0.9790000319480896,
+        "HTML": 0.9890000224113464,
+        "Java": 0.9600000381469727,
+        "PHP": 0.9500000476837158
+      },
+      "llm_test_accuracy": {
+        "C": 0.9530000686645508,
+        "Python": 0.9860000610351562,
+        "HTML": 0.9880000352859497,
+        "Java": 0.9640000462532043,
+        "PHP": 0.9550000429153442
+      },
+      "llm_top_1_test_accuracy": {
+        "C": 0.657,
+        "Python": 0.637,
+        "HTML": 0.714,
+        "Java": 0.624,
+        "PHP": 0.582
+      },
+      "llm_top_2_test_accuracy": {
+        "C": 0.658,
+        "Python": 0.664,
+        "HTML": 0.801,
+        "Java": 0.697,
+        "PHP": 0.64
+      },
+      "llm_top_5_test_accuracy": {
+        "C": 0.752,
+        "Python": 0.726,
+        "HTML": 0.928,
+        "Java": 0.728,
+        "PHP": 0.694
+      },
+      "sae_top_1_test_accuracy": {
+        "C": 0.609,
+        "Python": 0.736,
+        "HTML": 0.892,
+        "Java": 0.712,
+        "PHP": 0.916
+      },
+      "sae_top_2_test_accuracy": {
+        "C": 0.644,
+        "Python": 0.743,
+        "HTML": 0.889,
+        "Java": 0.712,
+        "PHP": 0.911
+      },
+      "sae_top_5_test_accuracy": {
+        "C": 0.725,
+        "Python": 0.796,
+        "HTML": 0.914,
+        "Java": 0.794,
+        "PHP": 0.914
+      }
+    },
+    "fancyzhx/ag_news_results": {
+      "sae_test_accuracy": {
+        "0": 0.9320000410079956,
+        "1": 0.9810000658035278,
+        "2": 0.9310000538825989,
+        "3": 0.9390000700950623
+      },
+      "llm_test_accuracy": {
+        "0": 0.9380000233650208,
+        "1": 0.9880000352859497,
+        "2": 0.9300000667572021,
+        "3": 0.9550000429153442
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.805,
+        "1": 0.67,
+        "2": 0.648,
+        "3": 0.666
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.794,
+        "1": 0.795,
+        "2": 0.686,
+        "3": 0.79
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.818,
+        "1": 0.867,
+        "2": 0.756,
+        "3": 0.836
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.688,
+        "1": 0.934,
+        "2": 0.74,
+        "3": 0.629
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.833,
+        "1": 0.93,
+        "2": 0.818,
+        "3": 0.657
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.858,
+        "1": 0.934,
+        "2": 0.827,
+        "3": 0.8
+      }
+    },
+    "Helsinki-NLP/europarl_results": {
+      "sae_test_accuracy": {
+        "en": 1.0,
+        "fr": 0.9950000643730164,
+        "de": 0.9970000386238098,
+        "es": 0.9930000305175781,
+        "nl": 0.9960000514984131
+      },
+      "llm_test_accuracy": {
+        "en": 1.0,
+        "fr": 1.0,
+        "de": 0.999000072479248,
+        "es": 1.0,
+        "nl": 0.999000072479248
+      },
+      "llm_top_1_test_accuracy": {
+        "en": 0.748,
+        "fr": 0.59,
+        "de": 0.754,
+        "es": 0.494,
+        "nl": 0.662
+      },
+      "llm_top_2_test_accuracy": {
+        "en": 0.822,
+        "fr": 0.596,
+        "de": 0.831,
+        "es": 0.91,
+        "nl": 0.746
+      },
+      "llm_top_5_test_accuracy": {
+        "en": 0.887,
+        "fr": 0.922,
+        "de": 0.909,
+        "es": 0.982,
+        "nl": 0.865
+      },
+      "sae_top_1_test_accuracy": {
+        "en": 0.709,
+        "fr": 0.582,
+        "de": 0.906,
+        "es": 0.75,
+        "nl": 0.889
+      },
+      "sae_top_2_test_accuracy": {
+        "en": 0.97,
+        "fr": 0.97,
+        "de": 0.909,
+        "es": 0.903,
+        "nl": 0.998
+      },
+      "sae_top_5_test_accuracy": {
+        "en": 0.999,
+        "fr": 0.989,
+        "de": 0.943,
+        "es": 0.985,
+        "nl": 0.999
+      }
+    }
+  }
+}

old_relu_eval_results/tpp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_0_custom_sae_eval_results.json

@@ -0,0 +1,414 @@
+{
+  "eval_type_id": "tpp",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": false,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "1fcdc3f3-fc8d-4e93-bfd2-d7725eac7b72",
+  "datetime_epoch_millis": 1738807905009,
+  "eval_result_metrics": {
+    "tpp_metrics": {
+      "tpp_threshold_2_total_metric": 0.005549997091293335,
+      "tpp_threshold_2_intended_diff_only": 0.008399999141693114,
+      "tpp_threshold_2_unintended_diff_only": 0.00285000205039978,
+      "tpp_threshold_5_total_metric": 0.00882500559091568,
+      "tpp_threshold_5_intended_diff_only": 0.01300000548362732,
+      "tpp_threshold_5_unintended_diff_only": 0.004174999892711639,
+      "tpp_threshold_10_total_metric": 0.020424994826316833,
+      "tpp_threshold_10_intended_diff_only": 0.02619999647140503,
+      "tpp_threshold_10_unintended_diff_only": 0.005775001645088196,
+      "tpp_threshold_20_total_metric": 0.030799996852874757,
+      "tpp_threshold_20_intended_diff_only": 0.036699998378753665,
+      "tpp_threshold_20_unintended_diff_only": 0.005900001525878907,
+      "tpp_threshold_50_total_metric": 0.07465000301599503,
+      "tpp_threshold_50_intended_diff_only": 0.08340000510215759,
+      "tpp_threshold_50_unintended_diff_only": 0.008750002086162566,
+      "tpp_threshold_100_total_metric": 0.14097500890493392,
+      "tpp_threshold_100_intended_diff_only": 0.15220001339912415,
+      "tpp_threshold_100_unintended_diff_only": 0.011225004494190217,
+      "tpp_threshold_500_total_metric": 0.41430001705884933,
+      "tpp_threshold_500_intended_diff_only": 0.4369000196456909,
+      "tpp_threshold_500_unintended_diff_only": 0.022600002586841583
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
+      "tpp_threshold_2_total_metric": 0.003849995136260986,
+      "tpp_threshold_2_intended_diff_only": 0.00559999942779541,
+      "tpp_threshold_2_unintended_diff_only": 0.0017500042915344239,
+      "tpp_threshold_5_total_metric": 0.006900015473365784,
+      "tpp_threshold_5_intended_diff_only": 0.009200012683868409,
+      "tpp_threshold_5_unintended_diff_only": 0.0022999972105026247,
+      "tpp_threshold_10_total_metric": 0.01449999213218689,
+      "tpp_threshold_10_intended_diff_only": 0.017199993133544922,
+      "tpp_threshold_10_unintended_diff_only": 0.0027000010013580322,
+      "tpp_threshold_20_total_metric": 0.02719999849796295,
+      "tpp_threshold_20_intended_diff_only": 0.029799997806549072,
+      "tpp_threshold_20_unintended_diff_only": 0.0025999993085861206,
+      "tpp_threshold_50_total_metric": 0.05915001034736633,
+      "tpp_threshold_50_intended_diff_only": 0.0634000062942505,
+      "tpp_threshold_50_unintended_diff_only": 0.004249995946884156,
+      "tpp_threshold_100_total_metric": 0.12605001628398896,
+      "tpp_threshold_100_intended_diff_only": 0.13100001811981202,
+      "tpp_threshold_100_unintended_diff_only": 0.004950001835823059,
+      "tpp_threshold_500_total_metric": 0.4442000240087509,
+      "tpp_threshold_500_intended_diff_only": 0.45640002489089965,
+      "tpp_threshold_500_unintended_diff_only": 0.012200000882148742
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
+      "tpp_threshold_2_total_metric": 0.007249999046325684,
+      "tpp_threshold_2_intended_diff_only": 0.01119999885559082,
+      "tpp_threshold_2_unintended_diff_only": 0.003949999809265137,
+      "tpp_threshold_5_total_metric": 0.010749995708465576,
+      "tpp_threshold_5_intended_diff_only": 0.01679999828338623,
+      "tpp_threshold_5_unintended_diff_only": 0.006050002574920654,
+      "tpp_threshold_10_total_metric": 0.026349997520446776,
+      "tpp_threshold_10_intended_diff_only": 0.03519999980926514,
+      "tpp_threshold_10_unintended_diff_only": 0.008850002288818359,
+      "tpp_threshold_20_total_metric": 0.03439999520778656,
+      "tpp_threshold_20_intended_diff_only": 0.04359999895095825,
+      "tpp_threshold_20_unintended_diff_only": 0.009200003743171693,
+      "tpp_threshold_50_total_metric": 0.09014999568462372,
+      "tpp_threshold_50_intended_diff_only": 0.1034000039100647,
+      "tpp_threshold_50_unintended_diff_only": 0.013250008225440979,
+      "tpp_threshold_100_total_metric": 0.1559000015258789,
+      "tpp_threshold_100_intended_diff_only": 0.17340000867843627,
+      "tpp_threshold_100_unintended_diff_only": 0.017500007152557374,
+      "tpp_threshold_500_total_metric": 0.38440001010894775,
+      "tpp_threshold_500_intended_diff_only": 0.4174000144004822,
+      "tpp_threshold_500_unintended_diff_only": 0.03300000429153442
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_0",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1": {
+      "0": {
+        "tpp_threshold_2_total_metric": 0.005750015377998352,
+        "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
+        "tpp_threshold_2_unintended_diff_only": 0.002250000834465027,
+        "tpp_threshold_5_total_metric": 0.013500049710273743,
+        "tpp_threshold_5_intended_diff_only": 0.016000032424926758,
+        "tpp_threshold_5_unintended_diff_only": 0.002499982714653015,
+        "tpp_threshold_10_total_metric": 0.013500019907951355,
+        "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_10_unintended_diff_only": 0.00449998676776886,
+        "tpp_threshold_20_total_metric": 0.03125002980232239,
+        "tpp_threshold_20_intended_diff_only": 0.0350000262260437,
+        "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_50_total_metric": 0.0712500512599945,
+        "tpp_threshold_50_intended_diff_only": 0.07500004768371582,
+        "tpp_threshold_50_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_100_total_metric": 0.1625000536441803,
+        "tpp_threshold_100_intended_diff_only": 0.16700005531311035,
+        "tpp_threshold_100_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_500_total_metric": 0.4487500488758087,
+        "tpp_threshold_500_intended_diff_only": 0.45500004291534424,
+        "tpp_threshold_500_unintended_diff_only": 0.0062499940395355225
+      },
+      "1": {
+        "tpp_threshold_2_total_metric": 0.0029999762773513794,
+        "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
+        "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224,
+        "tpp_threshold_5_total_metric": -0.0007500052452087402,
+        "tpp_threshold_5_intended_diff_only": 0.0,
+        "tpp_threshold_5_unintended_diff_only": 0.0007500052452087402,
+        "tpp_threshold_10_total_metric": 0.0034999698400497437,
+        "tpp_threshold_10_intended_diff_only": 0.0029999613761901855,
+        "tpp_threshold_10_unintended_diff_only": -0.0005000084638595581,
+        "tpp_threshold_20_total_metric": 0.0104999840259552,
+        "tpp_threshold_20_intended_diff_only": 0.014999985694885254,
+        "tpp_threshold_20_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_50_total_metric": 0.03200000524520874,
+        "tpp_threshold_50_intended_diff_only": 0.03700000047683716,
+        "tpp_threshold_50_unintended_diff_only": 0.004999995231628418,
+        "tpp_threshold_100_total_metric": 0.07999999821186066,
+        "tpp_threshold_100_intended_diff_only": 0.08700001239776611,
+        "tpp_threshold_100_unintended_diff_only": 0.0070000141859054565,
+        "tpp_threshold_500_total_metric": 0.43025001883506775,
+        "tpp_threshold_500_intended_diff_only": 0.4390000104904175,
+        "tpp_threshold_500_unintended_diff_only": 0.008749991655349731
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.0012499988079071045,
+        "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_2_unintended_diff_only": 0.002750009298324585,
+        "tpp_threshold_5_total_metric": 0.008750006556510925,
+        "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
+        "tpp_threshold_5_unintended_diff_only": 0.004250004887580872,
+        "tpp_threshold_10_total_metric": 0.01950003206729889,
+        "tpp_threshold_10_intended_diff_only": 0.021000027656555176,
+        "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866,
+        "tpp_threshold_20_total_metric": 0.028249993920326233,
+        "tpp_threshold_20_intended_diff_only": 0.02799999713897705,
+        "tpp_threshold_20_unintended_diff_only": -0.00024999678134918213,
+        "tpp_threshold_50_total_metric": 0.03624999523162842,
+        "tpp_threshold_50_intended_diff_only": 0.03799998760223389,
+        "tpp_threshold_50_unintended_diff_only": 0.0017499923706054688,
+        "tpp_threshold_100_total_metric": 0.08100000023841858,
+        "tpp_threshold_100_intended_diff_only": 0.08300000429153442,
+        "tpp_threshold_100_unintended_diff_only": 0.0020000040531158447,
+        "tpp_threshold_500_total_metric": 0.4345000237226486,
+        "tpp_threshold_500_intended_diff_only": 0.44700002670288086,
+        "tpp_threshold_500_unintended_diff_only": 0.012500002980232239
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.0015000104904174805,
+        "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
+        "tpp_threshold_2_unintended_diff_only": 0.0015000104904174805,
+        "tpp_threshold_5_total_metric": 0.0025000423192977905,
+        "tpp_threshold_5_intended_diff_only": 0.0020000338554382324,
+        "tpp_threshold_5_unintended_diff_only": -0.0005000084638595581,
+        "tpp_threshold_10_total_metric": 0.002499982714653015,
+        "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
+        "tpp_threshold_10_unintended_diff_only": 0.002500012516975403,
+        "tpp_threshold_20_total_metric": 0.00475001335144043,
+        "tpp_threshold_20_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_20_unintended_diff_only": -0.0007500052452087402,
+        "tpp_threshold_50_total_metric": 0.00950002670288086,
+        "tpp_threshold_50_intended_diff_only": 0.012000024318695068,
+        "tpp_threshold_50_unintended_diff_only": 0.002499997615814209,
+        "tpp_threshold_100_total_metric": 0.015250012278556824,
+        "tpp_threshold_100_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_100_unintended_diff_only": 0.002749994397163391,
+        "tpp_threshold_500_total_metric": 0.4517500102519989,
+        "tpp_threshold_500_intended_diff_only": 0.4620000123977661,
+        "tpp_threshold_500_unintended_diff_only": 0.010250002145767212
+      },
+      "9": {
+        "tpp_threshold_2_total_metric": 0.007749974727630615,
+        "tpp_threshold_2_intended_diff_only": 0.010999977588653564,
+        "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_5_total_metric": 0.0104999840259552,
+        "tpp_threshold_5_intended_diff_only": 0.014999985694885254,
+        "tpp_threshold_5_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_10_total_metric": 0.033499956130981445,
+        "tpp_threshold_10_intended_diff_only": 0.038999974727630615,
+        "tpp_threshold_10_unintended_diff_only": 0.00550001859664917,
+        "tpp_threshold_20_total_metric": 0.06124997138977051,
+        "tpp_threshold_20_intended_diff_only": 0.06699997186660767,
+        "tpp_threshold_20_unintended_diff_only": 0.005750000476837158,
+        "tpp_threshold_50_total_metric": 0.14674997329711914,
+        "tpp_threshold_50_intended_diff_only": 0.1549999713897705,
+        "tpp_threshold_50_unintended_diff_only": 0.008249998092651367,
+        "tpp_threshold_100_total_metric": 0.2915000170469284,
+        "tpp_threshold_100_intended_diff_only": 0.30000001192092896,
+        "tpp_threshold_100_unintended_diff_only": 0.00849999487400055,
+        "tpp_threshold_500_total_metric": 0.4557500183582306,
+        "tpp_threshold_500_intended_diff_only": 0.4790000319480896,
+        "tpp_threshold_500_unintended_diff_only": 0.02325001358985901
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5": {
+      "1": {
+        "tpp_threshold_2_total_metric": 0.012250036001205444,
+        "tpp_threshold_2_intended_diff_only": 0.016000032424926758,
+        "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_5_total_metric": 0.011750057339668274,
+        "tpp_threshold_5_intended_diff_only": 0.01500004529953003,
+        "tpp_threshold_5_unintended_diff_only": 0.0032499879598617554,
+        "tpp_threshold_10_total_metric": 0.012500032782554626,
+        "tpp_threshold_10_intended_diff_only": 0.021000027656555176,
+        "tpp_threshold_10_unintended_diff_only": 0.00849999487400055,
+        "tpp_threshold_20_total_metric": 0.022000029683113098,
+        "tpp_threshold_20_intended_diff_only": 0.03100001811981201,
+        "tpp_threshold_20_unintended_diff_only": 0.008999988436698914,
+        "tpp_threshold_50_total_metric": 0.06149999797344208,
+        "tpp_threshold_50_intended_diff_only": 0.0690000057220459,
+        "tpp_threshold_50_unintended_diff_only": 0.007500007748603821,
+        "tpp_threshold_100_total_metric": 0.10575002431869507,
+        "tpp_threshold_100_intended_diff_only": 0.11800003051757812,
+        "tpp_threshold_100_unintended_diff_only": 0.012250006198883057,
+        "tpp_threshold_500_total_metric": 0.41975003480911255,
+        "tpp_threshold_500_intended_diff_only": 0.44200003147125244,
+        "tpp_threshold_500_unintended_diff_only": 0.022249996662139893
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.0007499605417251587,
+        "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
+        "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983,
+        "tpp_threshold_5_total_metric": -0.013000041246414185,
+        "tpp_threshold_5_intended_diff_only": 0.0029999613761901855,
+        "tpp_threshold_5_unintended_diff_only": 0.01600000262260437,
+        "tpp_threshold_10_total_metric": 0.0022499561309814453,
+        "tpp_threshold_10_intended_diff_only": 0.01699995994567871,
+        "tpp_threshold_10_unintended_diff_only": 0.014750003814697266,
+        "tpp_threshold_20_total_metric": 0.0157499760389328,
+        "tpp_threshold_20_intended_diff_only": 0.02399998903274536,
+        "tpp_threshold_20_unintended_diff_only": 0.008250012993812561,
+        "tpp_threshold_50_total_metric": 0.05649995803833008,
+        "tpp_threshold_50_intended_diff_only": 0.08099997043609619,
+        "tpp_threshold_50_unintended_diff_only": 0.024500012397766113,
+        "tpp_threshold_100_total_metric": 0.12999998033046722,
+        "tpp_threshold_100_intended_diff_only": 0.15799999237060547,
+        "tpp_threshold_100_unintended_diff_only": 0.028000012040138245,
+        "tpp_threshold_500_total_metric": 0.3737499713897705,
+        "tpp_threshold_500_intended_diff_only": 0.4269999861717224,
+        "tpp_threshold_500_unintended_diff_only": 0.053250014781951904
+      },
+      "3": {
+        "tpp_threshold_2_total_metric": -0.0070000141859054565,
+        "tpp_threshold_2_intended_diff_only": -0.0040000081062316895,
+        "tpp_threshold_2_unintended_diff_only": 0.003000006079673767,
+        "tpp_threshold_5_total_metric": 0.004249989986419678,
+        "tpp_threshold_5_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_5_unintended_diff_only": -0.0002499818801879883,
+        "tpp_threshold_10_total_metric": 0.024749979376792908,
+        "tpp_threshold_10_intended_diff_only": 0.02899998426437378,
+        "tpp_threshold_10_unintended_diff_only": 0.004250004887580872,
+        "tpp_threshold_20_total_metric": 0.014499977231025696,
+        "tpp_threshold_20_intended_diff_only": 0.02399998903274536,
+        "tpp_threshold_20_unintended_diff_only": 0.009500011801719666,
+        "tpp_threshold_50_total_metric": 0.07250000536441803,
+        "tpp_threshold_50_intended_diff_only": 0.0820000171661377,
+        "tpp_threshold_50_unintended_diff_only": 0.009500011801719666,
+        "tpp_threshold_100_total_metric": 0.12725001573562622,
+        "tpp_threshold_100_intended_diff_only": 0.1420000195503235,
+        "tpp_threshold_100_unintended_diff_only": 0.014750003814697266,
+        "tpp_threshold_500_total_metric": 0.3969999998807907,
+        "tpp_threshold_500_intended_diff_only": 0.4259999990463257,
+        "tpp_threshold_500_unintended_diff_only": 0.028999999165534973
+      },
+      "5": {
+        "tpp_threshold_2_total_metric": -0.006999999284744263,
+        "tpp_threshold_2_intended_diff_only": -0.0040000081062316895,
+        "tpp_threshold_2_unintended_diff_only": 0.0029999911785125732,
+        "tpp_threshold_5_total_metric": -0.0015000104904174805,
+        "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
+        "tpp_threshold_5_unintended_diff_only": 0.0065000057220458984,
+        "tpp_threshold_10_total_metric": 0.007250010967254639,
+        "tpp_threshold_10_intended_diff_only": 0.017000019550323486,
+        "tpp_threshold_10_unintended_diff_only": 0.009750008583068848,
+        "tpp_threshold_20_total_metric": 0.02724999189376831,
+        "tpp_threshold_20_intended_diff_only": 0.03799998760223389,
+        "tpp_threshold_20_unintended_diff_only": 0.010749995708465576,
+        "tpp_threshold_50_total_metric": 0.07350002229213715,
+        "tpp_threshold_50_intended_diff_only": 0.08600002527236938,
+        "tpp_threshold_50_unintended_diff_only": 0.012500002980232239,
+        "tpp_threshold_100_total_metric": 0.13650000095367432,
+        "tpp_threshold_100_intended_diff_only": 0.15700000524520874,
+        "tpp_threshold_100_unintended_diff_only": 0.020500004291534424,
+        "tpp_threshold_500_total_metric": 0.3790000081062317,
+        "tpp_threshold_500_intended_diff_only": 0.41600000858306885,
+        "tpp_threshold_500_unintended_diff_only": 0.03700000047683716
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.037250012159347534,
+        "tpp_threshold_2_intended_diff_only": 0.046000003814697266,
+        "tpp_threshold_2_unintended_diff_only": 0.008749991655349731,
+        "tpp_threshold_5_total_metric": 0.052249982953071594,
+        "tpp_threshold_5_intended_diff_only": 0.05699998140335083,
+        "tpp_threshold_5_unintended_diff_only": 0.004749998450279236,
+        "tpp_threshold_10_total_metric": 0.08500000834465027,
+        "tpp_threshold_10_intended_diff_only": 0.09200000762939453,
+        "tpp_threshold_10_unintended_diff_only": 0.006999999284744263,
+        "tpp_threshold_20_total_metric": 0.0925000011920929,
+        "tpp_threshold_20_intended_diff_only": 0.10100001096725464,
+        "tpp_threshold_20_unintended_diff_only": 0.008500009775161743,
+        "tpp_threshold_50_total_metric": 0.18674999475479126,
+        "tpp_threshold_50_intended_diff_only": 0.19900000095367432,
+        "tpp_threshold_50_unintended_diff_only": 0.012250006198883057,
+        "tpp_threshold_100_total_metric": 0.2799999862909317,
+        "tpp_threshold_100_intended_diff_only": 0.2919999957084656,
+        "tpp_threshold_100_unintended_diff_only": 0.012000009417533875,
+        "tpp_threshold_500_total_metric": 0.3525000363588333,
+        "tpp_threshold_500_intended_diff_only": 0.3760000467300415,
+        "tpp_threshold_500_unintended_diff_only": 0.02350001037120819
+      }
+    }
+  }
+}

old_relu_eval_results/tpp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_1_custom_sae_eval_results.json

@@ -0,0 +1,414 @@
+{
+  "eval_type_id": "tpp",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": false,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "dca94803-7711-4df2-8ffe-a1d99bedfde5",
+  "datetime_epoch_millis": 1738808022044,
+  "eval_result_metrics": {
+    "tpp_metrics": {
+      "tpp_threshold_2_total_metric": 0.0034999996423721313,
+      "tpp_threshold_2_intended_diff_only": 0.00690000057220459,
+      "tpp_threshold_2_unintended_diff_only": 0.0034000009298324587,
+      "tpp_threshold_5_total_metric": 0.007625006139278412,
+      "tpp_threshold_5_intended_diff_only": 0.011600005626678466,
+      "tpp_threshold_5_unintended_diff_only": 0.003974999487400055,
+      "tpp_threshold_10_total_metric": 0.02060000002384186,
+      "tpp_threshold_10_intended_diff_only": 0.026100003719329835,
+      "tpp_threshold_10_unintended_diff_only": 0.005500003695487976,
+      "tpp_threshold_20_total_metric": 0.03362499922513962,
+      "tpp_threshold_20_intended_diff_only": 0.038699996471405027,
+      "tpp_threshold_20_unintended_diff_only": 0.005074997246265411,
+      "tpp_threshold_50_total_metric": 0.072200009226799,
+      "tpp_threshold_50_intended_diff_only": 0.08010000586509705,
+      "tpp_threshold_50_unintended_diff_only": 0.007899996638298035,
+      "tpp_threshold_100_total_metric": 0.1294749990105629,
+      "tpp_threshold_100_intended_diff_only": 0.14089999794960023,
+      "tpp_threshold_100_unintended_diff_only": 0.011424998939037322,
+      "tpp_threshold_500_total_metric": 0.40307500660419465,
+      "tpp_threshold_500_intended_diff_only": 0.4214000105857849,
+      "tpp_threshold_500_unintended_diff_only": 0.01832500398159027
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
+      "tpp_threshold_2_total_metric": 0.0038499802350997923,
+      "tpp_threshold_2_intended_diff_only": 0.005799984931945801,
+      "tpp_threshold_2_unintended_diff_only": 0.0019500046968460084,
+      "tpp_threshold_5_total_metric": 0.006150004267692566,
+      "tpp_threshold_5_intended_diff_only": 0.008399999141693116,
+      "tpp_threshold_5_unintended_diff_only": 0.0022499948740005494,
+      "tpp_threshold_10_total_metric": 0.017299994826316833,
+      "tpp_threshold_10_intended_diff_only": 0.019599997997283937,
+      "tpp_threshold_10_unintended_diff_only": 0.002300003170967102,
+      "tpp_threshold_20_total_metric": 0.03334999978542328,
+      "tpp_threshold_20_intended_diff_only": 0.035399997234344484,
+      "tpp_threshold_20_unintended_diff_only": 0.0020499974489212036,
+      "tpp_threshold_50_total_metric": 0.05845000743865967,
+      "tpp_threshold_50_intended_diff_only": 0.062400007247924806,
+      "tpp_threshold_50_unintended_diff_only": 0.003949999809265137,
+      "tpp_threshold_100_total_metric": 0.12464999556541442,
+      "tpp_threshold_100_intended_diff_only": 0.12999999523162842,
+      "tpp_threshold_100_unintended_diff_only": 0.00534999966621399,
+      "tpp_threshold_500_total_metric": 0.43650000989437104,
+      "tpp_threshold_500_intended_diff_only": 0.44520001411437987,
+      "tpp_threshold_500_unintended_diff_only": 0.00870000422000885
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
+      "tpp_threshold_2_total_metric": 0.0031500190496444704,
+      "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
+      "tpp_threshold_2_unintended_diff_only": 0.0048499971628189085,
+      "tpp_threshold_5_total_metric": 0.009100008010864257,
+      "tpp_threshold_5_intended_diff_only": 0.014800012111663818,
+      "tpp_threshold_5_unintended_diff_only": 0.00570000410079956,
+      "tpp_threshold_10_total_metric": 0.02390000522136688,
+      "tpp_threshold_10_intended_diff_only": 0.032600009441375734,
+      "tpp_threshold_10_unintended_diff_only": 0.00870000422000885,
+      "tpp_threshold_20_total_metric": 0.033899998664855956,
+      "tpp_threshold_20_intended_diff_only": 0.041999995708465576,
+      "tpp_threshold_20_unintended_diff_only": 0.008099997043609619,
+      "tpp_threshold_50_total_metric": 0.08595001101493835,
+      "tpp_threshold_50_intended_diff_only": 0.09780000448226929,
+      "tpp_threshold_50_unintended_diff_only": 0.011849993467330932,
+      "tpp_threshold_100_total_metric": 0.13430000245571136,
+      "tpp_threshold_100_intended_diff_only": 0.15180000066757202,
+      "tpp_threshold_100_unintended_diff_only": 0.017499998211860657,
+      "tpp_threshold_500_total_metric": 0.36965000331401826,
+      "tpp_threshold_500_intended_diff_only": 0.39760000705718995,
+      "tpp_threshold_500_unintended_diff_only": 0.027950003743171692
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_1",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1": {
+      "0": {
+        "tpp_threshold_2_total_metric": 0.005000025033950806,
+        "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
+        "tpp_threshold_2_unintended_diff_only": 0.0020000040531158447,
+        "tpp_threshold_5_total_metric": 0.010250017046928406,
+        "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
+        "tpp_threshold_5_unintended_diff_only": 0.002749994397163391,
+        "tpp_threshold_10_total_metric": 0.017250031232833862,
+        "tpp_threshold_10_intended_diff_only": 0.020000040531158447,
+        "tpp_threshold_10_unintended_diff_only": 0.002750009298324585,
+        "tpp_threshold_20_total_metric": 0.025500014424324036,
+        "tpp_threshold_20_intended_diff_only": 0.02799999713897705,
+        "tpp_threshold_20_unintended_diff_only": 0.002499982714653015,
+        "tpp_threshold_50_total_metric": 0.053750038146972656,
+        "tpp_threshold_50_intended_diff_only": 0.057000041007995605,
+        "tpp_threshold_50_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_100_total_metric": 0.1327500194311142,
+        "tpp_threshold_100_intended_diff_only": 0.1380000114440918,
+        "tpp_threshold_100_unintended_diff_only": 0.0052499920129776,
+        "tpp_threshold_500_total_metric": 0.43675003945827484,
+        "tpp_threshold_500_intended_diff_only": 0.44200003147125244,
+        "tpp_threshold_500_unintended_diff_only": 0.0052499920129776
+      },
+      "1": {
+        "tpp_threshold_2_total_metric": 0.0037499666213989258,
+        "tpp_threshold_2_intended_diff_only": 0.0029999613761901855,
+        "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402,
+        "tpp_threshold_5_total_metric": -0.0012499988079071045,
+        "tpp_threshold_5_intended_diff_only": 0.0,
+        "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045,
+        "tpp_threshold_10_total_metric": 0.005000010132789612,
+        "tpp_threshold_10_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_10_unintended_diff_only": -0.0010000020265579224,
+        "tpp_threshold_20_total_metric": 0.017249956727027893,
+        "tpp_threshold_20_intended_diff_only": 0.02199995517730713,
+        "tpp_threshold_20_unintended_diff_only": 0.004749998450279236,
+        "tpp_threshold_50_total_metric": 0.03124995529651642,
+        "tpp_threshold_50_intended_diff_only": 0.035999953746795654,
+        "tpp_threshold_50_unintended_diff_only": 0.004749998450279236,
+        "tpp_threshold_100_total_metric": 0.07649995386600494,
+        "tpp_threshold_100_intended_diff_only": 0.08199995756149292,
+        "tpp_threshold_100_unintended_diff_only": 0.005500003695487976,
+        "tpp_threshold_500_total_metric": 0.40824997425079346,
+        "tpp_threshold_500_intended_diff_only": 0.4169999957084656,
+        "tpp_threshold_500_unintended_diff_only": 0.00875002145767212
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.003749951720237732,
+        "tpp_threshold_2_intended_diff_only": 0.006999969482421875,
+        "tpp_threshold_2_unintended_diff_only": 0.003250017762184143,
+        "tpp_threshold_5_total_metric": 0.010749995708465576,
+        "tpp_threshold_5_intended_diff_only": 0.014999985694885254,
+        "tpp_threshold_5_unintended_diff_only": 0.004249989986419678,
+        "tpp_threshold_10_total_metric": 0.022749975323677063,
+        "tpp_threshold_10_intended_diff_only": 0.02499997615814209,
+        "tpp_threshold_10_unintended_diff_only": 0.002250000834465027,
+        "tpp_threshold_20_total_metric": 0.0339999794960022,
+        "tpp_threshold_20_intended_diff_only": 0.03299999237060547,
+        "tpp_threshold_20_unintended_diff_only": -0.0009999871253967285,
+        "tpp_threshold_50_total_metric": 0.03925001621246338,
+        "tpp_threshold_50_intended_diff_only": 0.04000002145767212,
+        "tpp_threshold_50_unintended_diff_only": 0.0007500052452087402,
+        "tpp_threshold_100_total_metric": 0.12300001084804535,
+        "tpp_threshold_100_intended_diff_only": 0.12400001287460327,
+        "tpp_threshold_100_unintended_diff_only": 0.0010000020265579224,
+        "tpp_threshold_500_total_metric": 0.43324999511241913,
+        "tpp_threshold_500_intended_diff_only": 0.4390000104904175,
+        "tpp_threshold_500_unintended_diff_only": 0.005750015377998352
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.002499997615814209,
+        "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_2_unintended_diff_only": 0.0015000104904174805,
+        "tpp_threshold_5_total_metric": 0.0030000507831573486,
+        "tpp_threshold_5_intended_diff_only": 0.0020000338554382324,
+        "tpp_threshold_5_unintended_diff_only": -0.0010000169277191162,
+        "tpp_threshold_10_total_metric": 0.0037499815225601196,
+        "tpp_threshold_10_intended_diff_only": 0.0059999823570251465,
+        "tpp_threshold_10_unintended_diff_only": 0.002250000834465027,
+        "tpp_threshold_20_total_metric": 0.008750036358833313,
+        "tpp_threshold_20_intended_diff_only": 0.00700002908706665,
+        "tpp_threshold_20_unintended_diff_only": -0.0017500072717666626,
+        "tpp_threshold_50_total_metric": 0.01150001585483551,
+        "tpp_threshold_50_intended_diff_only": 0.013000011444091797,
+        "tpp_threshold_50_unintended_diff_only": 0.0014999955892562866,
+        "tpp_threshold_100_total_metric": 0.013750031590461731,
+        "tpp_threshold_100_intended_diff_only": 0.017000019550323486,
+        "tpp_threshold_100_unintended_diff_only": 0.0032499879598617554,
+        "tpp_threshold_500_total_metric": 0.4442500174045563,
+        "tpp_threshold_500_intended_diff_only": 0.4490000009536743,
+        "tpp_threshold_500_unintended_diff_only": 0.004749983549118042
+      },
+      "9": {
+        "tpp_threshold_2_total_metric": 0.00424996018409729,
+        "tpp_threshold_2_intended_diff_only": 0.007999956607818604,
+        "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_5_total_metric": 0.007999956607818604,
+        "tpp_threshold_5_intended_diff_only": 0.011999964714050293,
+        "tpp_threshold_5_unintended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_10_total_metric": 0.03774997591972351,
+        "tpp_threshold_10_intended_diff_only": 0.042999982833862305,
+        "tpp_threshold_10_unintended_diff_only": 0.005250006914138794,
+        "tpp_threshold_20_total_metric": 0.08125001192092896,
+        "tpp_threshold_20_intended_diff_only": 0.08700001239776611,
+        "tpp_threshold_20_unintended_diff_only": 0.005750000476837158,
+        "tpp_threshold_50_total_metric": 0.15650001168251038,
+        "tpp_threshold_50_intended_diff_only": 0.16600000858306885,
+        "tpp_threshold_50_unintended_diff_only": 0.009499996900558472,
+        "tpp_threshold_100_total_metric": 0.2772499620914459,
+        "tpp_threshold_100_intended_diff_only": 0.2889999747276306,
+        "tpp_threshold_100_unintended_diff_only": 0.011750012636184692,
+        "tpp_threshold_500_total_metric": 0.46000002324581146,
+        "tpp_threshold_500_intended_diff_only": 0.4790000319480896,
+        "tpp_threshold_500_unintended_diff_only": 0.019000008702278137
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5": {
+      "1": {
+        "tpp_threshold_2_total_metric": 0.009750023484230042,
+        "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
+        "tpp_threshold_2_unintended_diff_only": 0.0032499879598617554,
+        "tpp_threshold_5_total_metric": 0.007750034332275391,
+        "tpp_threshold_5_intended_diff_only": 0.010000050067901611,
+        "tpp_threshold_5_unintended_diff_only": 0.0022500157356262207,
+        "tpp_threshold_10_total_metric": 0.012500032782554626,
+        "tpp_threshold_10_intended_diff_only": 0.021000027656555176,
+        "tpp_threshold_10_unintended_diff_only": 0.00849999487400055,
+        "tpp_threshold_20_total_metric": 0.014500007033348083,
+        "tpp_threshold_20_intended_diff_only": 0.023000001907348633,
+        "tpp_threshold_20_unintended_diff_only": 0.00849999487400055,
+        "tpp_threshold_50_total_metric": 0.05975005030632019,
+        "tpp_threshold_50_intended_diff_only": 0.06600004434585571,
+        "tpp_threshold_50_unintended_diff_only": 0.0062499940395355225,
+        "tpp_threshold_100_total_metric": 0.09250001609325409,
+        "tpp_threshold_100_intended_diff_only": 0.10600000619888306,
+        "tpp_threshold_100_unintended_diff_only": 0.013499990105628967,
+        "tpp_threshold_500_total_metric": 0.3750000149011612,
+        "tpp_threshold_500_intended_diff_only": 0.3970000147819519,
+        "tpp_threshold_500_unintended_diff_only": 0.02199999988079071
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.0045000165700912476,
+        "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
+        "tpp_threshold_2_unintended_diff_only": 0.00449998676776886,
+        "tpp_threshold_5_total_metric": -0.002500012516975403,
+        "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
+        "tpp_threshold_5_unintended_diff_only": 0.0155000239610672,
+        "tpp_threshold_10_total_metric": 0.010499954223632812,
+        "tpp_threshold_10_intended_diff_only": 0.02199995517730713,
+        "tpp_threshold_10_unintended_diff_only": 0.011500000953674316,
+        "tpp_threshold_20_total_metric": 0.031749993562698364,
+        "tpp_threshold_20_intended_diff_only": 0.03700000047683716,
+        "tpp_threshold_20_unintended_diff_only": 0.005250006914138794,
+        "tpp_threshold_50_total_metric": 0.06824998557567596,
+        "tpp_threshold_50_intended_diff_only": 0.08899998664855957,
+        "tpp_threshold_50_unintended_diff_only": 0.020750001072883606,
+        "tpp_threshold_100_total_metric": 0.11849997937679291,
+        "tpp_threshold_100_intended_diff_only": 0.14499998092651367,
+        "tpp_threshold_100_unintended_diff_only": 0.026500001549720764,
+        "tpp_threshold_500_total_metric": 0.37724997103214264,
+        "tpp_threshold_500_intended_diff_only": 0.4179999828338623,
+        "tpp_threshold_500_unintended_diff_only": 0.040750011801719666
+      },
+      "3": {
+        "tpp_threshold_2_total_metric": -0.008250012993812561,
+        "tpp_threshold_2_intended_diff_only": -0.004999995231628418,
+        "tpp_threshold_2_unintended_diff_only": 0.003250017762184143,
+        "tpp_threshold_5_total_metric": -0.002750024199485779,
+        "tpp_threshold_5_intended_diff_only": -0.0020000338554382324,
+        "tpp_threshold_5_unintended_diff_only": 0.0007499903440475464,
+        "tpp_threshold_10_total_metric": 0.018250003457069397,
+        "tpp_threshold_10_intended_diff_only": 0.022000014781951904,
+        "tpp_threshold_10_unintended_diff_only": 0.0037500113248825073,
+        "tpp_threshold_20_total_metric": 0.010499969124794006,
+        "tpp_threshold_20_intended_diff_only": 0.01699995994567871,
+        "tpp_threshold_20_unintended_diff_only": 0.006499990820884705,
+        "tpp_threshold_50_total_metric": 0.06024998426437378,
+        "tpp_threshold_50_intended_diff_only": 0.06699997186660767,
+        "tpp_threshold_50_unintended_diff_only": 0.006749987602233887,
+        "tpp_threshold_100_total_metric": 0.11424997448921204,
+        "tpp_threshold_100_intended_diff_only": 0.12699997425079346,
+        "tpp_threshold_100_unintended_diff_only": 0.012749999761581421,
+        "tpp_threshold_500_total_metric": 0.3764999955892563,
+        "tpp_threshold_500_intended_diff_only": 0.39800000190734863,
+        "tpp_threshold_500_unintended_diff_only": 0.021500006318092346
+      },
+      "5": {
+        "tpp_threshold_2_total_metric": -0.0057499706745147705,
+        "tpp_threshold_2_intended_diff_only": -0.001999974250793457,
+        "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_5_total_metric": 0.009500041604042053,
+        "tpp_threshold_5_intended_diff_only": 0.016000032424926758,
+        "tpp_threshold_5_unintended_diff_only": 0.006499990820884705,
+        "tpp_threshold_10_total_metric": 0.012000009417533875,
+        "tpp_threshold_10_intended_diff_only": 0.022000014781951904,
+        "tpp_threshold_10_unintended_diff_only": 0.01000000536441803,
+        "tpp_threshold_20_total_metric": 0.027250006794929504,
+        "tpp_threshold_20_intended_diff_only": 0.03700000047683716,
+        "tpp_threshold_20_unintended_diff_only": 0.009749993681907654,
+        "tpp_threshold_50_total_metric": 0.07799999415874481,
+        "tpp_threshold_50_intended_diff_only": 0.08899998664855957,
+        "tpp_threshold_50_unintended_diff_only": 0.010999992489814758,
+        "tpp_threshold_100_total_metric": 0.11225004494190216,
+        "tpp_threshold_100_intended_diff_only": 0.13100004196166992,
+        "tpp_threshold_100_unintended_diff_only": 0.01874999701976776,
+        "tpp_threshold_500_total_metric": 0.36875003576278687,
+        "tpp_threshold_500_intended_diff_only": 0.40000003576278687,
+        "tpp_threshold_500_unintended_diff_only": 0.03125
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.015500038862228394,
+        "tpp_threshold_2_intended_diff_only": 0.025000035762786865,
+        "tpp_threshold_2_unintended_diff_only": 0.009499996900558472,
+        "tpp_threshold_5_total_metric": 0.03350000083446503,
+        "tpp_threshold_5_intended_diff_only": 0.03700000047683716,
+        "tpp_threshold_5_unintended_diff_only": 0.0034999996423721313,
+        "tpp_threshold_10_total_metric": 0.0662500262260437,
+        "tpp_threshold_10_intended_diff_only": 0.07600003480911255,
+        "tpp_threshold_10_unintended_diff_only": 0.009750008583068848,
+        "tpp_threshold_20_total_metric": 0.08550001680850983,
+        "tpp_threshold_20_intended_diff_only": 0.09600001573562622,
+        "tpp_threshold_20_unintended_diff_only": 0.010499998927116394,
+        "tpp_threshold_50_total_metric": 0.16350004076957703,
+        "tpp_threshold_50_intended_diff_only": 0.17800003290176392,
+        "tpp_threshold_50_unintended_diff_only": 0.01449999213218689,
+        "tpp_threshold_100_total_metric": 0.23399999737739563,
+        "tpp_threshold_100_intended_diff_only": 0.25,
+        "tpp_threshold_100_unintended_diff_only": 0.01600000262260437,
+        "tpp_threshold_500_total_metric": 0.35074999928474426,
+        "tpp_threshold_500_intended_diff_only": 0.375,
+        "tpp_threshold_500_unintended_diff_only": 0.024250000715255737
+      }
+    }
+  }
+}

old_relu_eval_results/tpp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_2_custom_sae_eval_results.json

@@ -0,0 +1,414 @@
+{
+  "eval_type_id": "tpp",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": false,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "bafd917a-113b-4ade-93b9-fc280e1e64e8",
+  "datetime_epoch_millis": 1738808371674,
+  "eval_result_metrics": {
+    "tpp_metrics": {
+      "tpp_threshold_2_total_metric": 0.004999996721744537,
+      "tpp_threshold_2_intended_diff_only": 0.008099997043609619,
+      "tpp_threshold_2_unintended_diff_only": 0.003100000321865082,
+      "tpp_threshold_5_total_metric": 0.004899995028972625,
+      "tpp_threshold_5_intended_diff_only": 0.008599996566772461,
+      "tpp_threshold_5_unintended_diff_only": 0.0037000015377998356,
+      "tpp_threshold_10_total_metric": 0.017099998891353607,
+      "tpp_threshold_10_intended_diff_only": 0.02160000205039978,
+      "tpp_threshold_10_unintended_diff_only": 0.004500003159046173,
+      "tpp_threshold_20_total_metric": 0.028300000727176665,
+      "tpp_threshold_20_intended_diff_only": 0.03340000510215759,
+      "tpp_threshold_20_unintended_diff_only": 0.0051000043749809265,
+      "tpp_threshold_50_total_metric": 0.059400005638599394,
+      "tpp_threshold_50_intended_diff_only": 0.0662000060081482,
+      "tpp_threshold_50_unintended_diff_only": 0.0068000003695487974,
+      "tpp_threshold_100_total_metric": 0.10572500675916671,
+      "tpp_threshold_100_intended_diff_only": 0.11560000777244568,
+      "tpp_threshold_100_unintended_diff_only": 0.00987500101327896,
+      "tpp_threshold_500_total_metric": 0.3575750187039375,
+      "tpp_threshold_500_intended_diff_only": 0.3726000189781189,
+      "tpp_threshold_500_unintended_diff_only": 0.015025000274181365
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
+      "tpp_threshold_2_total_metric": 0.005649995803833008,
+      "tpp_threshold_2_intended_diff_only": 0.0075999975204467775,
+      "tpp_threshold_2_unintended_diff_only": 0.0019500017166137694,
+      "tpp_threshold_5_total_metric": 0.005199992656707763,
+      "tpp_threshold_5_intended_diff_only": 0.0075999975204467775,
+      "tpp_threshold_5_unintended_diff_only": 0.0024000048637390138,
+      "tpp_threshold_10_total_metric": 0.014100000262260437,
+      "tpp_threshold_10_intended_diff_only": 0.01640000343322754,
+      "tpp_threshold_10_unintended_diff_only": 0.002300003170967102,
+      "tpp_threshold_20_total_metric": 0.028600004315376282,
+      "tpp_threshold_20_intended_diff_only": 0.031000006198883056,
+      "tpp_threshold_20_unintended_diff_only": 0.002400001883506775,
+      "tpp_threshold_50_total_metric": 0.053450003266334534,
+      "tpp_threshold_50_intended_diff_only": 0.05700000524520874,
+      "tpp_threshold_50_unintended_diff_only": 0.0035500019788742066,
+      "tpp_threshold_100_total_metric": 0.10355001091957092,
+      "tpp_threshold_100_intended_diff_only": 0.10860000848770142,
+      "tpp_threshold_100_unintended_diff_only": 0.005049997568130493,
+      "tpp_threshold_500_total_metric": 0.39755001962184905,
+      "tpp_threshold_500_intended_diff_only": 0.4052000164985657,
+      "tpp_threshold_500_unintended_diff_only": 0.007649996876716613
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
+      "tpp_threshold_2_total_metric": 0.004349997639656067,
+      "tpp_threshold_2_intended_diff_only": 0.008599996566772461,
+      "tpp_threshold_2_unintended_diff_only": 0.004249998927116394,
+      "tpp_threshold_5_total_metric": 0.0045999974012374874,
+      "tpp_threshold_5_intended_diff_only": 0.009599995613098145,
+      "tpp_threshold_5_unintended_diff_only": 0.004999998211860657,
+      "tpp_threshold_10_total_metric": 0.020099997520446777,
+      "tpp_threshold_10_intended_diff_only": 0.026800000667572023,
+      "tpp_threshold_10_unintended_diff_only": 0.006700003147125244,
+      "tpp_threshold_20_total_metric": 0.02799999713897705,
+      "tpp_threshold_20_intended_diff_only": 0.03580000400543213,
+      "tpp_threshold_20_unintended_diff_only": 0.007800006866455078,
+      "tpp_threshold_50_total_metric": 0.06535000801086426,
+      "tpp_threshold_50_intended_diff_only": 0.07540000677108764,
+      "tpp_threshold_50_unintended_diff_only": 0.010049998760223389,
+      "tpp_threshold_100_total_metric": 0.1079000025987625,
+      "tpp_threshold_100_intended_diff_only": 0.12260000705718994,
+      "tpp_threshold_100_unintended_diff_only": 0.014700004458427429,
+      "tpp_threshold_500_total_metric": 0.317600017786026,
+      "tpp_threshold_500_intended_diff_only": 0.3400000214576721,
+      "tpp_threshold_500_unintended_diff_only": 0.02240000367164612
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_2",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1": {
+      "0": {
+        "tpp_threshold_2_total_metric": 0.01075001060962677,
+        "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
+        "tpp_threshold_2_unintended_diff_only": 0.002250000834465027,
+        "tpp_threshold_5_total_metric": 0.01600000262260437,
+        "tpp_threshold_5_intended_diff_only": 0.018999993801116943,
+        "tpp_threshold_5_unintended_diff_only": 0.0029999911785125732,
+        "tpp_threshold_10_total_metric": 0.013500034809112549,
+        "tpp_threshold_10_intended_diff_only": 0.016000032424926758,
+        "tpp_threshold_10_unintended_diff_only": 0.002499997615814209,
+        "tpp_threshold_20_total_metric": 0.023750022053718567,
+        "tpp_threshold_20_intended_diff_only": 0.027000010013580322,
+        "tpp_threshold_20_unintended_diff_only": 0.0032499879598617554,
+        "tpp_threshold_50_total_metric": 0.057750046253204346,
+        "tpp_threshold_50_intended_diff_only": 0.061000049114227295,
+        "tpp_threshold_50_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_100_total_metric": 0.11575004458427429,
+        "tpp_threshold_100_intended_diff_only": 0.12200003862380981,
+        "tpp_threshold_100_unintended_diff_only": 0.0062499940395355225,
+        "tpp_threshold_500_total_metric": 0.4100000709295273,
+        "tpp_threshold_500_intended_diff_only": 0.41200006008148193,
+        "tpp_threshold_500_unintended_diff_only": 0.001999989151954651
+      },
+      "1": {
+        "tpp_threshold_2_total_metric": 0.0014999955892562866,
+        "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
+        "tpp_threshold_2_unintended_diff_only": -0.0005000084638595581,
+        "tpp_threshold_5_total_metric": -0.005250021815299988,
+        "tpp_threshold_5_intended_diff_only": -0.0040000081062316895,
+        "tpp_threshold_5_unintended_diff_only": 0.0012500137090682983,
+        "tpp_threshold_10_total_metric": -0.0005000382661819458,
+        "tpp_threshold_10_intended_diff_only": -0.001000046730041504,
+        "tpp_threshold_10_unintended_diff_only": -0.0005000084638595581,
+        "tpp_threshold_20_total_metric": 0.006749957799911499,
+        "tpp_threshold_20_intended_diff_only": 0.010999977588653564,
+        "tpp_threshold_20_unintended_diff_only": 0.004250019788742065,
+        "tpp_threshold_50_total_metric": 0.02424997091293335,
+        "tpp_threshold_50_intended_diff_only": 0.02899998426437378,
+        "tpp_threshold_50_unintended_diff_only": 0.00475001335144043,
+        "tpp_threshold_100_total_metric": 0.047499969601631165,
+        "tpp_threshold_100_intended_diff_only": 0.05199998617172241,
+        "tpp_threshold_100_unintended_diff_only": 0.0045000165700912476,
+        "tpp_threshold_500_total_metric": 0.2900000065565109,
+        "tpp_threshold_500_intended_diff_only": 0.29500001668930054,
+        "tpp_threshold_500_unintended_diff_only": 0.005000010132789612
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.00875002145767212,
+        "tpp_threshold_2_intended_diff_only": 0.012000024318695068,
+        "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_5_total_metric": 0.007250010967254639,
+        "tpp_threshold_5_intended_diff_only": 0.012000024318695068,
+        "tpp_threshold_5_unintended_diff_only": 0.00475001335144043,
+        "tpp_threshold_10_total_metric": 0.024500012397766113,
+        "tpp_threshold_10_intended_diff_only": 0.026000022888183594,
+        "tpp_threshold_10_unintended_diff_only": 0.0015000104904174805,
+        "tpp_threshold_20_total_metric": 0.03749999403953552,
+        "tpp_threshold_20_intended_diff_only": 0.03700000047683716,
+        "tpp_threshold_20_unintended_diff_only": -0.0004999935626983643,
+        "tpp_threshold_50_total_metric": 0.04449997842311859,
+        "tpp_threshold_50_intended_diff_only": 0.042999982833862305,
+        "tpp_threshold_50_unintended_diff_only": -0.0014999955892562866,
+        "tpp_threshold_100_total_metric": 0.08000004291534424,
+        "tpp_threshold_100_intended_diff_only": 0.08100003004074097,
+        "tpp_threshold_100_unintended_diff_only": 0.0009999871253967285,
+        "tpp_threshold_500_total_metric": 0.4047499895095825,
+        "tpp_threshold_500_intended_diff_only": 0.4129999876022339,
+        "tpp_threshold_500_unintended_diff_only": 0.008249998092651367
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.0029999911785125732,
+        "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162,
+        "tpp_threshold_5_total_metric": 0.003750026226043701,
+        "tpp_threshold_5_intended_diff_only": 0.003000020980834961,
+        "tpp_threshold_5_unintended_diff_only": -0.0007500052452087402,
+        "tpp_threshold_10_total_metric": 0.002249985933303833,
+        "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
+        "tpp_threshold_10_unintended_diff_only": 0.002750009298324585,
+        "tpp_threshold_20_total_metric": 0.008000046014785767,
+        "tpp_threshold_20_intended_diff_only": 0.00700002908706665,
+        "tpp_threshold_20_unintended_diff_only": -0.0010000169277191162,
+        "tpp_threshold_50_total_metric": 0.012750029563903809,
+        "tpp_threshold_50_intended_diff_only": 0.017000019550323486,
+        "tpp_threshold_50_unintended_diff_only": 0.004249989986419678,
+        "tpp_threshold_100_total_metric": 0.04250001907348633,
+        "tpp_threshold_100_intended_diff_only": 0.046000003814697266,
+        "tpp_threshold_100_unintended_diff_only": 0.0034999847412109375,
+        "tpp_threshold_500_total_metric": 0.42250001430511475,
+        "tpp_threshold_500_intended_diff_only": 0.4300000071525574,
+        "tpp_threshold_500_unintended_diff_only": 0.007499992847442627
+      },
+      "9": {
+        "tpp_threshold_2_total_metric": 0.00424996018409729,
+        "tpp_threshold_2_intended_diff_only": 0.007999956607818604,
+        "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_5_total_metric": 0.004249945282936096,
+        "tpp_threshold_5_intended_diff_only": 0.007999956607818604,
+        "tpp_threshold_5_unintended_diff_only": 0.0037500113248825073,
+        "tpp_threshold_10_total_metric": 0.030750006437301636,
+        "tpp_threshold_10_intended_diff_only": 0.03600001335144043,
+        "tpp_threshold_10_unintended_diff_only": 0.005250006914138794,
+        "tpp_threshold_20_total_metric": 0.06700000166893005,
+        "tpp_threshold_20_intended_diff_only": 0.07300001382827759,
+        "tpp_threshold_20_unintended_diff_only": 0.006000012159347534,
+        "tpp_threshold_50_total_metric": 0.12799999117851257,
+        "tpp_threshold_50_intended_diff_only": 0.13499999046325684,
+        "tpp_threshold_50_unintended_diff_only": 0.006999999284744263,
+        "tpp_threshold_100_total_metric": 0.2319999784231186,
+        "tpp_threshold_100_intended_diff_only": 0.24199998378753662,
+        "tpp_threshold_100_unintended_diff_only": 0.01000000536441803,
+        "tpp_threshold_500_total_metric": 0.4605000168085098,
+        "tpp_threshold_500_intended_diff_only": 0.47600001096725464,
+        "tpp_threshold_500_unintended_diff_only": 0.015499994158744812
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5": {
+      "1": {
+        "tpp_threshold_2_total_metric": 0.013250023126602173,
+        "tpp_threshold_2_intended_diff_only": 0.017000019550323486,
+        "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_5_total_metric": 0.011250004172325134,
+        "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_5_unintended_diff_only": 0.002749994397163391,
+        "tpp_threshold_10_total_metric": 0.01075001060962677,
+        "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_10_unintended_diff_only": 0.007249996066093445,
+        "tpp_threshold_20_total_metric": 0.015000015497207642,
+        "tpp_threshold_20_intended_diff_only": 0.021000027656555176,
+        "tpp_threshold_20_unintended_diff_only": 0.006000012159347534,
+        "tpp_threshold_50_total_metric": 0.03725004196166992,
+        "tpp_threshold_50_intended_diff_only": 0.04300004243850708,
+        "tpp_threshold_50_unintended_diff_only": 0.005750000476837158,
+        "tpp_threshold_100_total_metric": 0.0612499862909317,
+        "tpp_threshold_100_intended_diff_only": 0.07400000095367432,
+        "tpp_threshold_100_unintended_diff_only": 0.012750014662742615,
+        "tpp_threshold_500_total_metric": 0.28825002908706665,
+        "tpp_threshold_500_intended_diff_only": 0.2990000247955322,
+        "tpp_threshold_500_unintended_diff_only": 0.010749995708465576
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.008249983191490173,
+        "tpp_threshold_2_intended_diff_only": 0.010999977588653564,
+        "tpp_threshold_2_unintended_diff_only": 0.002749994397163391,
+        "tpp_threshold_5_total_metric": -0.0034999996423721313,
+        "tpp_threshold_5_intended_diff_only": 0.009000003337860107,
+        "tpp_threshold_5_unintended_diff_only": 0.012500002980232239,
+        "tpp_threshold_10_total_metric": 0.015499994158744812,
+        "tpp_threshold_10_intended_diff_only": 0.02399998903274536,
+        "tpp_threshold_10_unintended_diff_only": 0.00849999487400055,
+        "tpp_threshold_20_total_metric": 0.02699996531009674,
+        "tpp_threshold_20_intended_diff_only": 0.03299999237060547,
+        "tpp_threshold_20_unintended_diff_only": 0.006000027060508728,
+        "tpp_threshold_50_total_metric": 0.0677499920129776,
+        "tpp_threshold_50_intended_diff_only": 0.07999998331069946,
+        "tpp_threshold_50_unintended_diff_only": 0.012249991297721863,
+        "tpp_threshold_100_total_metric": 0.10849995911121368,
+        "tpp_threshold_100_intended_diff_only": 0.12699997425079346,
+        "tpp_threshold_100_unintended_diff_only": 0.018500015139579773,
+        "tpp_threshold_500_total_metric": 0.3362499922513962,
+        "tpp_threshold_500_intended_diff_only": 0.3669999837875366,
+        "tpp_threshold_500_unintended_diff_only": 0.030749991536140442
+      },
+      "3": {
+        "tpp_threshold_2_total_metric": -0.005500048398971558,
+        "tpp_threshold_2_intended_diff_only": -0.0020000338554382324,
+        "tpp_threshold_2_unintended_diff_only": 0.003500014543533325,
+        "tpp_threshold_5_total_metric": 0.0037499964237213135,
+        "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
+        "tpp_threshold_5_unintended_diff_only": 0.0012499988079071045,
+        "tpp_threshold_10_total_metric": 0.0209999680519104,
+        "tpp_threshold_10_intended_diff_only": 0.02399998903274536,
+        "tpp_threshold_10_unintended_diff_only": 0.003000020980834961,
+        "tpp_threshold_20_total_metric": 0.010250002145767212,
+        "tpp_threshold_20_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_20_unintended_diff_only": 0.007750004529953003,
+        "tpp_threshold_50_total_metric": 0.06074999272823334,
+        "tpp_threshold_50_intended_diff_only": 0.06999999284744263,
+        "tpp_threshold_50_unintended_diff_only": 0.00925000011920929,
+        "tpp_threshold_100_total_metric": 0.08075001835823059,
+        "tpp_threshold_100_intended_diff_only": 0.09600001573562622,
+        "tpp_threshold_100_unintended_diff_only": 0.01524999737739563,
+        "tpp_threshold_500_total_metric": 0.3087500035762787,
+        "tpp_threshold_500_intended_diff_only": 0.3320000171661377,
+        "tpp_threshold_500_unintended_diff_only": 0.02325001358985901
+      },
+      "5": {
+        "tpp_threshold_2_total_metric": -0.006499990820884705,
+        "tpp_threshold_2_intended_diff_only": -0.0040000081062316895,
+        "tpp_threshold_2_unintended_diff_only": 0.002499982714653015,
+        "tpp_threshold_5_total_metric": -0.0052499920129776,
+        "tpp_threshold_5_intended_diff_only": 0.0,
+        "tpp_threshold_5_unintended_diff_only": 0.0052499920129776,
+        "tpp_threshold_10_total_metric": -0.0022500157356262207,
+        "tpp_threshold_10_intended_diff_only": 0.0059999823570251465,
+        "tpp_threshold_10_unintended_diff_only": 0.008249998092651367,
+        "tpp_threshold_20_total_metric": 0.013750016689300537,
+        "tpp_threshold_20_intended_diff_only": 0.023000001907348633,
+        "tpp_threshold_20_unintended_diff_only": 0.009249985218048096,
+        "tpp_threshold_50_total_metric": 0.03574998676776886,
+        "tpp_threshold_50_intended_diff_only": 0.046999990940093994,
+        "tpp_threshold_50_unintended_diff_only": 0.011250004172325134,
+        "tpp_threshold_100_total_metric": 0.08550001680850983,
+        "tpp_threshold_100_intended_diff_only": 0.10100001096725464,
+        "tpp_threshold_100_unintended_diff_only": 0.015499994158744812,
+        "tpp_threshold_500_total_metric": 0.3072500377893448,
+        "tpp_threshold_500_intended_diff_only": 0.33500003814697266,
+        "tpp_threshold_500_unintended_diff_only": 0.02775000035762787
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.01225002110004425,
+        "tpp_threshold_2_intended_diff_only": 0.021000027656555176,
+        "tpp_threshold_2_unintended_diff_only": 0.008750006556510925,
+        "tpp_threshold_5_total_metric": 0.016749978065490723,
+        "tpp_threshold_5_intended_diff_only": 0.019999980926513672,
+        "tpp_threshold_5_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_10_total_metric": 0.055500030517578125,
+        "tpp_threshold_10_intended_diff_only": 0.06200003623962402,
+        "tpp_threshold_10_unintended_diff_only": 0.0065000057220458984,
+        "tpp_threshold_20_total_metric": 0.07399998605251312,
+        "tpp_threshold_20_intended_diff_only": 0.08399999141693115,
+        "tpp_threshold_20_unintended_diff_only": 0.01000000536441803,
+        "tpp_threshold_50_total_metric": 0.12525002658367157,
+        "tpp_threshold_50_intended_diff_only": 0.13700002431869507,
+        "tpp_threshold_50_unintended_diff_only": 0.011749997735023499,
+        "tpp_threshold_100_total_metric": 0.20350003242492676,
+        "tpp_threshold_100_intended_diff_only": 0.21500003337860107,
+        "tpp_threshold_100_unintended_diff_only": 0.011500000953674316,
+        "tpp_threshold_500_total_metric": 0.3475000262260437,
+        "tpp_threshold_500_intended_diff_only": 0.3670000433921814,
+        "tpp_threshold_500_unintended_diff_only": 0.019500017166137695
+      }
+    }
+  }
+}

old_relu_eval_results/tpp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_3_custom_sae_eval_results.json

@@ -0,0 +1,414 @@
+{
+  "eval_type_id": "tpp",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": false,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "a9df393e-0561-4737-9c16-8f03033409cb",
+  "datetime_epoch_millis": 1738808254874,
+  "eval_result_metrics": {
+    "tpp_metrics": {
+      "tpp_threshold_2_total_metric": 0.0035749942064285277,
+      "tpp_threshold_2_intended_diff_only": 0.0061999976634979255,
+      "tpp_threshold_2_unintended_diff_only": 0.002625003457069397,
+      "tpp_threshold_5_total_metric": 0.005500002205371857,
+      "tpp_threshold_5_intended_diff_only": 0.009100002050399781,
+      "tpp_threshold_5_unintended_diff_only": 0.003599999845027924,
+      "tpp_threshold_10_total_metric": 0.01294999271631241,
+      "tpp_threshold_10_intended_diff_only": 0.017399996519088745,
+      "tpp_threshold_10_unintended_diff_only": 0.004450003802776336,
+      "tpp_threshold_20_total_metric": 0.021499992907047273,
+      "tpp_threshold_20_intended_diff_only": 0.0262999951839447,
+      "tpp_threshold_20_unintended_diff_only": 0.00480000227689743,
+      "tpp_threshold_50_total_metric": 0.043899996578693395,
+      "tpp_threshold_50_intended_diff_only": 0.04889999628067017,
+      "tpp_threshold_50_unintended_diff_only": 0.004999999701976777,
+      "tpp_threshold_100_total_metric": 0.07275000959634781,
+      "tpp_threshold_100_intended_diff_only": 0.08150001168251038,
+      "tpp_threshold_100_unintended_diff_only": 0.008750002086162566,
+      "tpp_threshold_500_total_metric": 0.2600750118494034,
+      "tpp_threshold_500_intended_diff_only": 0.27110000848770144,
+      "tpp_threshold_500_unintended_diff_only": 0.011024996638298035
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
+      "tpp_threshold_2_total_metric": 0.003349986672401428,
+      "tpp_threshold_2_intended_diff_only": 0.0053999900817871095,
+      "tpp_threshold_2_unintended_diff_only": 0.002050003409385681,
+      "tpp_threshold_5_total_metric": 0.006400004029273987,
+      "tpp_threshold_5_intended_diff_only": 0.008800005912780762,
+      "tpp_threshold_5_unintended_diff_only": 0.002400001883506775,
+      "tpp_threshold_10_total_metric": 0.011899998784065247,
+      "tpp_threshold_10_intended_diff_only": 0.014800000190734863,
+      "tpp_threshold_10_unintended_diff_only": 0.0029000014066696165,
+      "tpp_threshold_20_total_metric": 0.023699989914894103,
+      "tpp_threshold_20_intended_diff_only": 0.02659999132156372,
+      "tpp_threshold_20_unintended_diff_only": 0.0029000014066696165,
+      "tpp_threshold_50_total_metric": 0.04629998803138733,
+      "tpp_threshold_50_intended_diff_only": 0.049799990653991696,
+      "tpp_threshold_50_unintended_diff_only": 0.0035000026226043703,
+      "tpp_threshold_100_total_metric": 0.07975000441074372,
+      "tpp_threshold_100_intended_diff_only": 0.08480000495910645,
+      "tpp_threshold_100_unintended_diff_only": 0.005050000548362732,
+      "tpp_threshold_500_total_metric": 0.31270001232624056,
+      "tpp_threshold_500_intended_diff_only": 0.3202000021934509,
+      "tpp_threshold_500_unintended_diff_only": 0.007499989867210388
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
+      "tpp_threshold_2_total_metric": 0.0038000017404556273,
+      "tpp_threshold_2_intended_diff_only": 0.007000005245208741,
+      "tpp_threshold_2_unintended_diff_only": 0.003200003504753113,
+      "tpp_threshold_5_total_metric": 0.004600000381469726,
+      "tpp_threshold_5_intended_diff_only": 0.009399998188018798,
+      "tpp_threshold_5_unintended_diff_only": 0.004799997806549073,
+      "tpp_threshold_10_total_metric": 0.01399998664855957,
+      "tpp_threshold_10_intended_diff_only": 0.019999992847442628,
+      "tpp_threshold_10_unintended_diff_only": 0.006000006198883056,
+      "tpp_threshold_20_total_metric": 0.01929999589920044,
+      "tpp_threshold_20_intended_diff_only": 0.025999999046325682,
+      "tpp_threshold_20_unintended_diff_only": 0.006700003147125244,
+      "tpp_threshold_50_total_metric": 0.04150000512599945,
+      "tpp_threshold_50_intended_diff_only": 0.048000001907348634,
+      "tpp_threshold_50_unintended_diff_only": 0.0064999967813491825,
+      "tpp_threshold_100_total_metric": 0.0657500147819519,
+      "tpp_threshold_100_intended_diff_only": 0.07820001840591431,
+      "tpp_threshold_100_unintended_diff_only": 0.012450003623962402,
+      "tpp_threshold_500_total_metric": 0.20745001137256622,
+      "tpp_threshold_500_intended_diff_only": 0.22200001478195192,
+      "tpp_threshold_500_unintended_diff_only": 0.01455000340938568
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_3",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1": {
+      "0": {
+        "tpp_threshold_2_total_metric": 0.007749989628791809,
+        "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
+        "tpp_threshold_2_unintended_diff_only": 0.002250000834465027,
+        "tpp_threshold_5_total_metric": 0.017250046133995056,
+        "tpp_threshold_5_intended_diff_only": 0.020000040531158447,
+        "tpp_threshold_5_unintended_diff_only": 0.002749994397163391,
+        "tpp_threshold_10_total_metric": 0.014750003814697266,
+        "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_20_total_metric": 0.023250028491020203,
+        "tpp_threshold_20_intended_diff_only": 0.026000022888183594,
+        "tpp_threshold_20_unintended_diff_only": 0.002749994397163391,
+        "tpp_threshold_50_total_metric": 0.04175001382827759,
+        "tpp_threshold_50_intended_diff_only": 0.04500001668930054,
+        "tpp_threshold_50_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_100_total_metric": 0.07125002145767212,
+        "tpp_threshold_100_intended_diff_only": 0.07700002193450928,
+        "tpp_threshold_100_unintended_diff_only": 0.005750000476837158,
+        "tpp_threshold_500_total_metric": 0.24525001645088196,
+        "tpp_threshold_500_intended_diff_only": 0.24500000476837158,
+        "tpp_threshold_500_unintended_diff_only": -0.000250011682510376
+      },
+      "1": {
+        "tpp_threshold_2_total_metric": 0.0029999613761901855,
+        "tpp_threshold_2_intended_diff_only": 0.0029999613761901855,
+        "tpp_threshold_2_unintended_diff_only": 0.0,
+        "tpp_threshold_5_total_metric": -0.0010000169277191162,
+        "tpp_threshold_5_intended_diff_only": 0.0009999871253967285,
+        "tpp_threshold_5_unintended_diff_only": 0.0020000040531158447,
+        "tpp_threshold_10_total_metric": 0.00925000011920929,
+        "tpp_threshold_10_intended_diff_only": 0.009000003337860107,
+        "tpp_threshold_10_unintended_diff_only": -0.00024999678134918213,
+        "tpp_threshold_20_total_metric": 0.010249972343444824,
+        "tpp_threshold_20_intended_diff_only": 0.014999985694885254,
+        "tpp_threshold_20_unintended_diff_only": 0.00475001335144043,
+        "tpp_threshold_50_total_metric": 0.025749951601028442,
+        "tpp_threshold_50_intended_diff_only": 0.030999958515167236,
+        "tpp_threshold_50_unintended_diff_only": 0.005250006914138794,
+        "tpp_threshold_100_total_metric": 0.05324995517730713,
+        "tpp_threshold_100_intended_diff_only": 0.05799996852874756,
+        "tpp_threshold_100_unintended_diff_only": 0.00475001335144043,
+        "tpp_threshold_500_total_metric": 0.26924997568130493,
+        "tpp_threshold_500_intended_diff_only": 0.2789999842643738,
+        "tpp_threshold_500_unintended_diff_only": 0.009750008583068848
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.0027499794960021973,
+        "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
+        "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_5_total_metric": 0.006249964237213135,
+        "tpp_threshold_5_intended_diff_only": 0.010999977588653564,
+        "tpp_threshold_5_unintended_diff_only": 0.00475001335144043,
+        "tpp_threshold_10_total_metric": 0.021499991416931152,
+        "tpp_threshold_10_intended_diff_only": 0.02399998903274536,
+        "tpp_threshold_10_unintended_diff_only": 0.002499997615814209,
+        "tpp_threshold_20_total_metric": 0.03274999558925629,
+        "tpp_threshold_20_intended_diff_only": 0.03299999237060547,
+        "tpp_threshold_20_unintended_diff_only": 0.00024999678134918213,
+        "tpp_threshold_50_total_metric": 0.03449997305870056,
+        "tpp_threshold_50_intended_diff_only": 0.0339999794960022,
+        "tpp_threshold_50_unintended_diff_only": -0.0004999935626983643,
+        "tpp_threshold_100_total_metric": 0.05425000190734863,
+        "tpp_threshold_100_intended_diff_only": 0.0559999942779541,
+        "tpp_threshold_100_unintended_diff_only": 0.0017499923706054688,
+        "tpp_threshold_500_total_metric": 0.3055000454187393,
+        "tpp_threshold_500_intended_diff_only": 0.312000036239624,
+        "tpp_threshold_500_unintended_diff_only": 0.006499990820884705
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.0010000169277191162,
+        "tpp_threshold_2_intended_diff_only": 0.0020000338554382324,
+        "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162,
+        "tpp_threshold_5_total_metric": 0.003000035881996155,
+        "tpp_threshold_5_intended_diff_only": 0.0020000338554382324,
+        "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224,
+        "tpp_threshold_10_total_metric": 0.0010000020265579224,
+        "tpp_threshold_10_intended_diff_only": 0.004999995231628418,
+        "tpp_threshold_10_unintended_diff_only": 0.003999993205070496,
+        "tpp_threshold_20_total_metric": 0.005499988794326782,
+        "tpp_threshold_20_intended_diff_only": 0.0059999823570251465,
+        "tpp_threshold_20_unintended_diff_only": 0.0004999935626983643,
+        "tpp_threshold_50_total_metric": 0.006500035524368286,
+        "tpp_threshold_50_intended_diff_only": 0.012000024318695068,
+        "tpp_threshold_50_unintended_diff_only": 0.005499988794326782,
+        "tpp_threshold_100_total_metric": 0.04650004208087921,
+        "tpp_threshold_100_intended_diff_only": 0.053000032901763916,
+        "tpp_threshold_100_unintended_diff_only": 0.006499990820884705,
+        "tpp_threshold_500_total_metric": 0.30650003254413605,
+        "tpp_threshold_500_intended_diff_only": 0.3190000057220459,
+        "tpp_threshold_500_unintended_diff_only": 0.012499973177909851
+      },
+      "9": {
+        "tpp_threshold_2_total_metric": 0.002249985933303833,
+        "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
+        "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_5_total_metric": 0.006499990820884705,
+        "tpp_threshold_5_intended_diff_only": 0.009999990463256836,
+        "tpp_threshold_5_unintended_diff_only": 0.0034999996423721313,
+        "tpp_threshold_10_total_metric": 0.012999996542930603,
+        "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_10_unintended_diff_only": 0.005000010132789612,
+        "tpp_threshold_20_total_metric": 0.046749964356422424,
+        "tpp_threshold_20_intended_diff_only": 0.05299997329711914,
+        "tpp_threshold_20_unintended_diff_only": 0.006250008940696716,
+        "tpp_threshold_50_total_metric": 0.12299996614456177,
+        "tpp_threshold_50_intended_diff_only": 0.12699997425079346,
+        "tpp_threshold_50_unintended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_100_total_metric": 0.17350000143051147,
+        "tpp_threshold_100_intended_diff_only": 0.18000000715255737,
+        "tpp_threshold_100_unintended_diff_only": 0.0065000057220458984,
+        "tpp_threshold_500_total_metric": 0.43699999153614044,
+        "tpp_threshold_500_intended_diff_only": 0.44599997997283936,
+        "tpp_threshold_500_unintended_diff_only": 0.008999988436698914
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5": {
+      "1": {
+        "tpp_threshold_2_total_metric": 0.01175004243850708,
+        "tpp_threshold_2_intended_diff_only": 0.01500004529953003,
+        "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_5_total_metric": 0.011750012636184692,
+        "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_5_unintended_diff_only": 0.002249985933303833,
+        "tpp_threshold_10_total_metric": 0.007500052452087402,
+        "tpp_threshold_10_intended_diff_only": 0.01500004529953003,
+        "tpp_threshold_10_unintended_diff_only": 0.007499992847442627,
+        "tpp_threshold_20_total_metric": 0.011250004172325134,
+        "tpp_threshold_20_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_20_unintended_diff_only": 0.006750002503395081,
+        "tpp_threshold_50_total_metric": 0.03525002300739288,
+        "tpp_threshold_50_intended_diff_only": 0.03600001335144043,
+        "tpp_threshold_50_unintended_diff_only": 0.0007499903440475464,
+        "tpp_threshold_100_total_metric": 0.03375004231929779,
+        "tpp_threshold_100_intended_diff_only": 0.04300004243850708,
+        "tpp_threshold_100_unintended_diff_only": 0.00925000011920929,
+        "tpp_threshold_500_total_metric": 0.15350006520748138,
+        "tpp_threshold_500_intended_diff_only": 0.16200006008148193,
+        "tpp_threshold_500_unintended_diff_only": 0.00849999487400055
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.0024999380111694336,
+        "tpp_threshold_2_intended_diff_only": 0.0029999613761901855,
+        "tpp_threshold_2_unintended_diff_only": 0.000500023365020752,
+        "tpp_threshold_5_total_metric": -0.011750012636184692,
+        "tpp_threshold_5_intended_diff_only": 0.0009999871253967285,
+        "tpp_threshold_5_unintended_diff_only": 0.012749999761581421,
+        "tpp_threshold_10_total_metric": 0.0024999380111694336,
+        "tpp_threshold_10_intended_diff_only": 0.007999956607818604,
+        "tpp_threshold_10_unintended_diff_only": 0.00550001859664917,
+        "tpp_threshold_20_total_metric": 0.007499963045120239,
+        "tpp_threshold_20_intended_diff_only": 0.011999964714050293,
+        "tpp_threshold_20_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_50_total_metric": 0.021749988198280334,
+        "tpp_threshold_50_intended_diff_only": 0.03299999237060547,
+        "tpp_threshold_50_unintended_diff_only": 0.011250004172325134,
+        "tpp_threshold_100_total_metric": 0.04649996757507324,
+        "tpp_threshold_100_intended_diff_only": 0.06599998474121094,
+        "tpp_threshold_100_unintended_diff_only": 0.019500017166137695,
+        "tpp_threshold_500_total_metric": 0.195499986410141,
+        "tpp_threshold_500_intended_diff_only": 0.21299999952316284,
+        "tpp_threshold_500_unintended_diff_only": 0.01750001311302185
+      },
+      "3": {
+        "tpp_threshold_2_total_metric": -0.003500014543533325,
+        "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
+        "tpp_threshold_2_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_5_total_metric": 0.004499956965446472,
+        "tpp_threshold_5_intended_diff_only": 0.006999969482421875,
+        "tpp_threshold_5_unintended_diff_only": 0.002500012516975403,
+        "tpp_threshold_10_total_metric": 0.019999995827674866,
+        "tpp_threshold_10_intended_diff_only": 0.023000001907348633,
+        "tpp_threshold_10_unintended_diff_only": 0.003000006079673767,
+        "tpp_threshold_20_total_metric": 0.008000001311302185,
+        "tpp_threshold_20_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_20_unintended_diff_only": 0.00599999725818634,
+        "tpp_threshold_50_total_metric": 0.03449995815753937,
+        "tpp_threshold_50_intended_diff_only": 0.034999966621398926,
+        "tpp_threshold_50_unintended_diff_only": 0.0005000084638595581,
+        "tpp_threshold_100_total_metric": 0.06475000083446503,
+        "tpp_threshold_100_intended_diff_only": 0.07400000095367432,
+        "tpp_threshold_100_unintended_diff_only": 0.00925000011920929,
+        "tpp_threshold_500_total_metric": 0.18399998545646667,
+        "tpp_threshold_500_intended_diff_only": 0.19900000095367432,
+        "tpp_threshold_500_unintended_diff_only": 0.015000015497207642
+      },
+      "5": {
+        "tpp_threshold_2_total_metric": -0.004999980330467224,
+        "tpp_threshold_2_intended_diff_only": -0.0009999871253967285,
+        "tpp_threshold_2_unintended_diff_only": 0.003999993205070496,
+        "tpp_threshold_5_total_metric": -0.002499982714653015,
+        "tpp_threshold_5_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_5_unintended_diff_only": 0.006499990820884705,
+        "tpp_threshold_10_total_metric": -0.002750009298324585,
+        "tpp_threshold_10_intended_diff_only": 0.0059999823570251465,
+        "tpp_threshold_10_unintended_diff_only": 0.008749991655349731,
+        "tpp_threshold_20_total_metric": 0.003000020980834961,
+        "tpp_threshold_20_intended_diff_only": 0.012000024318695068,
+        "tpp_threshold_20_unintended_diff_only": 0.009000003337860107,
+        "tpp_threshold_50_total_metric": 0.027250006794929504,
+        "tpp_threshold_50_intended_diff_only": 0.03700000047683716,
+        "tpp_threshold_50_unintended_diff_only": 0.009749993681907654,
+        "tpp_threshold_100_total_metric": 0.04350002110004425,
+        "tpp_threshold_100_intended_diff_only": 0.058000028133392334,
+        "tpp_threshold_100_unintended_diff_only": 0.014500007033348083,
+        "tpp_threshold_500_total_metric": 0.18325002491474152,
+        "tpp_threshold_500_intended_diff_only": 0.20200002193450928,
+        "tpp_threshold_500_unintended_diff_only": 0.01874999701976776
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.013250023126602173,
+        "tpp_threshold_2_intended_diff_only": 0.017000019550323486,
+        "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_5_total_metric": 0.021000027656555176,
+        "tpp_threshold_5_intended_diff_only": 0.021000027656555176,
+        "tpp_threshold_5_unintended_diff_only": 0.0,
+        "tpp_threshold_10_total_metric": 0.042749956250190735,
+        "tpp_threshold_10_intended_diff_only": 0.04799997806549072,
+        "tpp_threshold_10_unintended_diff_only": 0.005250021815299988,
+        "tpp_threshold_20_total_metric": 0.06674998998641968,
+        "tpp_threshold_20_intended_diff_only": 0.07400000095367432,
+        "tpp_threshold_20_unintended_diff_only": 0.007250010967254639,
+        "tpp_threshold_50_total_metric": 0.08875004947185516,
+        "tpp_threshold_50_intended_diff_only": 0.09900003671646118,
+        "tpp_threshold_50_unintended_diff_only": 0.010249987244606018,
+        "tpp_threshold_100_total_metric": 0.1402500420808792,
+        "tpp_threshold_100_intended_diff_only": 0.15000003576278687,
+        "tpp_threshold_100_unintended_diff_only": 0.009749993681907654,
+        "tpp_threshold_500_total_metric": 0.32099999487400055,
+        "tpp_threshold_500_intended_diff_only": 0.33399999141693115,
+        "tpp_threshold_500_unintended_diff_only": 0.012999996542930603
+      }
+    }
+  }
+}

old_relu_eval_results/tpp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_4_custom_sae_eval_results.json

@@ -0,0 +1,414 @@
+{
+  "eval_type_id": "tpp",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": false,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "d8d50b3f-efea-453e-80ea-09e656823d9c",
+  "datetime_epoch_millis": 1738808485682,
+  "eval_result_metrics": {
+    "tpp_metrics": {
+      "tpp_threshold_2_total_metric": 0.003474992513656616,
+      "tpp_threshold_2_intended_diff_only": 0.005799990892410278,
+      "tpp_threshold_2_unintended_diff_only": 0.002324998378753662,
+      "tpp_threshold_5_total_metric": 0.0039999991655349735,
+      "tpp_threshold_5_intended_diff_only": 0.006999999284744263,
+      "tpp_threshold_5_unintended_diff_only": 0.0030000001192092896,
+      "tpp_threshold_10_total_metric": 0.01015000194311142,
+      "tpp_threshold_10_intended_diff_only": 0.013400000333786011,
+      "tpp_threshold_10_unintended_diff_only": 0.0032499983906745912,
+      "tpp_threshold_20_total_metric": 0.016299988329410552,
+      "tpp_threshold_20_intended_diff_only": 0.019599992036819457,
+      "tpp_threshold_20_unintended_diff_only": 0.003300003707408905,
+      "tpp_threshold_50_total_metric": 0.03060000091791153,
+      "tpp_threshold_50_intended_diff_only": 0.03460000157356262,
+      "tpp_threshold_50_unintended_diff_only": 0.0040000006556510925,
+      "tpp_threshold_100_total_metric": 0.047974994778633116,
+      "tpp_threshold_100_intended_diff_only": 0.054599994421005243,
+      "tpp_threshold_100_unintended_diff_only": 0.0066249996423721315,
+      "tpp_threshold_500_total_metric": 0.18647501319646836,
+      "tpp_threshold_500_intended_diff_only": 0.19520001411437987,
+      "tpp_threshold_500_unintended_diff_only": 0.00872500091791153
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
+      "tpp_threshold_2_total_metric": 0.0038499891757965087,
+      "tpp_threshold_2_intended_diff_only": 0.005599987506866455,
+      "tpp_threshold_2_unintended_diff_only": 0.0017499983310699462,
+      "tpp_threshold_5_total_metric": 0.005899989604949951,
+      "tpp_threshold_5_intended_diff_only": 0.008799993991851806,
+      "tpp_threshold_5_unintended_diff_only": 0.0029000043869018555,
+      "tpp_threshold_10_total_metric": 0.008850002288818359,
+      "tpp_threshold_10_intended_diff_only": 0.01119999885559082,
+      "tpp_threshold_10_unintended_diff_only": 0.002349996566772461,
+      "tpp_threshold_20_total_metric": 0.01929998993873596,
+      "tpp_threshold_20_intended_diff_only": 0.021199989318847656,
+      "tpp_threshold_20_unintended_diff_only": 0.0018999993801116944,
+      "tpp_threshold_50_total_metric": 0.03360001742839813,
+      "tpp_threshold_50_intended_diff_only": 0.03660001754760742,
+      "tpp_threshold_50_unintended_diff_only": 0.0030000001192092896,
+      "tpp_threshold_100_total_metric": 0.05629999041557312,
+      "tpp_threshold_100_intended_diff_only": 0.06019998788833618,
+      "tpp_threshold_100_unintended_diff_only": 0.0038999974727630614,
+      "tpp_threshold_500_total_metric": 0.23835001289844512,
+      "tpp_threshold_500_intended_diff_only": 0.2446000099182129,
+      "tpp_threshold_500_unintended_diff_only": 0.006249997019767761
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
+      "tpp_threshold_2_total_metric": 0.003099995851516724,
+      "tpp_threshold_2_intended_diff_only": 0.005999994277954101,
+      "tpp_threshold_2_unintended_diff_only": 0.002899998426437378,
+      "tpp_threshold_5_total_metric": 0.0021000087261199953,
+      "tpp_threshold_5_intended_diff_only": 0.005200004577636719,
+      "tpp_threshold_5_unintended_diff_only": 0.003099995851516724,
+      "tpp_threshold_10_total_metric": 0.01145000159740448,
+      "tpp_threshold_10_intended_diff_only": 0.015600001811981202,
+      "tpp_threshold_10_unintended_diff_only": 0.004150000214576721,
+      "tpp_threshold_20_total_metric": 0.013299986720085144,
+      "tpp_threshold_20_intended_diff_only": 0.01799999475479126,
+      "tpp_threshold_20_unintended_diff_only": 0.004700008034706116,
+      "tpp_threshold_50_total_metric": 0.027599984407424928,
+      "tpp_threshold_50_intended_diff_only": 0.03259998559951782,
+      "tpp_threshold_50_unintended_diff_only": 0.005000001192092896,
+      "tpp_threshold_100_total_metric": 0.039649999141693114,
+      "tpp_threshold_100_intended_diff_only": 0.049000000953674315,
+      "tpp_threshold_100_unintended_diff_only": 0.009350001811981201,
+      "tpp_threshold_500_total_metric": 0.13460001349449158,
+      "tpp_threshold_500_intended_diff_only": 0.14580001831054687,
+      "tpp_threshold_500_unintended_diff_only": 0.011200004816055298
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_4",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1": {
+      "0": {
+        "tpp_threshold_2_total_metric": 0.0070000141859054565,
+        "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
+        "tpp_threshold_2_unintended_diff_only": 0.001999989151954651,
+        "tpp_threshold_5_total_metric": 0.014750033617019653,
+        "tpp_threshold_5_intended_diff_only": 0.017000019550323486,
+        "tpp_threshold_5_unintended_diff_only": 0.002249985933303833,
+        "tpp_threshold_10_total_metric": 0.005250036716461182,
+        "tpp_threshold_10_intended_diff_only": 0.00700002908706665,
+        "tpp_threshold_10_unintended_diff_only": 0.0017499923706054688,
+        "tpp_threshold_20_total_metric": 0.015250027179718018,
+        "tpp_threshold_20_intended_diff_only": 0.017000019550323486,
+        "tpp_threshold_20_unintended_diff_only": 0.0017499923706054688,
+        "tpp_threshold_50_total_metric": 0.02700003981590271,
+        "tpp_threshold_50_intended_diff_only": 0.030000030994415283,
+        "tpp_threshold_50_unintended_diff_only": 0.0029999911785125732,
+        "tpp_threshold_100_total_metric": 0.04625000059604645,
+        "tpp_threshold_100_intended_diff_only": 0.050999999046325684,
+        "tpp_threshold_100_unintended_diff_only": 0.004749998450279236,
+        "tpp_threshold_500_total_metric": 0.18425005674362183,
+        "tpp_threshold_500_intended_diff_only": 0.18700003623962402,
+        "tpp_threshold_500_unintended_diff_only": 0.0027499794960021973
+      },
+      "1": {
+        "tpp_threshold_2_total_metric": 0.0027499794960021973,
+        "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
+        "tpp_threshold_2_unintended_diff_only": -0.0007500052452087402,
+        "tpp_threshold_5_total_metric": -0.0025000721216201782,
+        "tpp_threshold_5_intended_diff_only": -0.001000046730041504,
+        "tpp_threshold_5_unintended_diff_only": 0.0015000253915786743,
+        "tpp_threshold_10_total_metric": 0.00475001335144043,
+        "tpp_threshold_10_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_10_unintended_diff_only": -0.0007500052452087402,
+        "tpp_threshold_20_total_metric": 0.0029999911785125732,
+        "tpp_threshold_20_intended_diff_only": 0.0059999823570251465,
+        "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732,
+        "tpp_threshold_50_total_metric": 0.02799999713897705,
+        "tpp_threshold_50_intended_diff_only": 0.03200000524520874,
+        "tpp_threshold_50_unintended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_100_total_metric": 0.04924996197223663,
+        "tpp_threshold_100_intended_diff_only": 0.05399996042251587,
+        "tpp_threshold_100_unintended_diff_only": 0.004749998450279236,
+        "tpp_threshold_500_total_metric": 0.164249986410141,
+        "tpp_threshold_500_intended_diff_only": 0.171999990940094,
+        "tpp_threshold_500_unintended_diff_only": 0.007750004529953003
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.005499988794326782,
+        "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
+        "tpp_threshold_2_unintended_diff_only": 0.003500014543533325,
+        "tpp_threshold_5_total_metric": 0.008000001311302185,
+        "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
+        "tpp_threshold_5_unintended_diff_only": 0.005000010132789612,
+        "tpp_threshold_10_total_metric": 0.020500019192695618,
+        "tpp_threshold_10_intended_diff_only": 0.023000001907348633,
+        "tpp_threshold_10_unintended_diff_only": 0.002499982714653015,
+        "tpp_threshold_20_total_metric": 0.029999971389770508,
+        "tpp_threshold_20_intended_diff_only": 0.029999971389770508,
+        "tpp_threshold_20_unintended_diff_only": 0.0,
+        "tpp_threshold_50_total_metric": 0.028249993920326233,
+        "tpp_threshold_50_intended_diff_only": 0.02799999713897705,
+        "tpp_threshold_50_unintended_diff_only": -0.00024999678134918213,
+        "tpp_threshold_100_total_metric": 0.040000006556510925,
+        "tpp_threshold_100_intended_diff_only": 0.04100000858306885,
+        "tpp_threshold_100_unintended_diff_only": 0.0010000020265579224,
+        "tpp_threshold_500_total_metric": 0.21049997210502625,
+        "tpp_threshold_500_intended_diff_only": 0.21399998664855957,
+        "tpp_threshold_500_unintended_diff_only": 0.003500014543533325
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.00024999678134918213,
+        "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
+        "tpp_threshold_2_unintended_diff_only": 0.0007499903440475464,
+        "tpp_threshold_5_total_metric": 0.0012500286102294922,
+        "tpp_threshold_5_intended_diff_only": 0.003000020980834961,
+        "tpp_threshold_5_unintended_diff_only": 0.0017499923706054688,
+        "tpp_threshold_10_total_metric": 0.002249985933303833,
+        "tpp_threshold_10_intended_diff_only": 0.0059999823570251465,
+        "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_20_total_metric": 0.005749985575675964,
+        "tpp_threshold_20_intended_diff_only": 0.0059999823570251465,
+        "tpp_threshold_20_unintended_diff_only": 0.00024999678134918213,
+        "tpp_threshold_50_total_metric": 0.00575004518032074,
+        "tpp_threshold_50_intended_diff_only": 0.01100003719329834,
+        "tpp_threshold_50_unintended_diff_only": 0.0052499920129776,
+        "tpp_threshold_100_total_metric": 0.026749998331069946,
+        "tpp_threshold_100_intended_diff_only": 0.02899998426437378,
+        "tpp_threshold_100_unintended_diff_only": 0.002249985933303833,
+        "tpp_threshold_500_total_metric": 0.2757500410079956,
+        "tpp_threshold_500_intended_diff_only": 0.2850000262260437,
+        "tpp_threshold_500_unintended_diff_only": 0.009249985218048096
+      },
+      "9": {
+        "tpp_threshold_2_total_metric": 0.0037499666213989258,
+        "tpp_threshold_2_intended_diff_only": 0.006999969482421875,
+        "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_5_total_metric": 0.007999956607818604,
+        "tpp_threshold_5_intended_diff_only": 0.011999964714050293,
+        "tpp_threshold_5_unintended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_10_total_metric": 0.011499956250190735,
+        "tpp_threshold_10_intended_diff_only": 0.015999972820281982,
+        "tpp_threshold_10_unintended_diff_only": 0.0045000165700912476,
+        "tpp_threshold_20_total_metric": 0.04249997437000275,
+        "tpp_threshold_20_intended_diff_only": 0.046999990940093994,
+        "tpp_threshold_20_unintended_diff_only": 0.0045000165700912476,
+        "tpp_threshold_50_total_metric": 0.07900001108646393,
+        "tpp_threshold_50_intended_diff_only": 0.0820000171661377,
+        "tpp_threshold_50_unintended_diff_only": 0.003000006079673767,
+        "tpp_threshold_100_total_metric": 0.11924998462200165,
+        "tpp_threshold_100_intended_diff_only": 0.12599998712539673,
+        "tpp_threshold_100_unintended_diff_only": 0.006750002503395081,
+        "tpp_threshold_500_total_metric": 0.357000008225441,
+        "tpp_threshold_500_intended_diff_only": 0.36500000953674316,
+        "tpp_threshold_500_unintended_diff_only": 0.008000001311302185
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5": {
+      "1": {
+        "tpp_threshold_2_total_metric": 0.010500013828277588,
+        "tpp_threshold_2_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_2_unintended_diff_only": 0.0034999847412109375,
+        "tpp_threshold_5_total_metric": 0.009250059723854065,
+        "tpp_threshold_5_intended_diff_only": 0.010000050067901611,
+        "tpp_threshold_5_unintended_diff_only": 0.0007499903440475464,
+        "tpp_threshold_10_total_metric": 0.0065000057220458984,
+        "tpp_threshold_10_intended_diff_only": 0.013000011444091797,
+        "tpp_threshold_10_unintended_diff_only": 0.0065000057220458984,
+        "tpp_threshold_20_total_metric": 0.008249998092651367,
+        "tpp_threshold_20_intended_diff_only": 0.013000011444091797,
+        "tpp_threshold_20_unintended_diff_only": 0.00475001335144043,
+        "tpp_threshold_50_total_metric": 0.02350001037120819,
+        "tpp_threshold_50_intended_diff_only": 0.023000001907348633,
+        "tpp_threshold_50_unintended_diff_only": -0.0005000084638595581,
+        "tpp_threshold_100_total_metric": 0.016500025987625122,
+        "tpp_threshold_100_intended_diff_only": 0.025000035762786865,
+        "tpp_threshold_100_unintended_diff_only": 0.008500009775161743,
+        "tpp_threshold_500_total_metric": 0.07200004160404205,
+        "tpp_threshold_500_intended_diff_only": 0.07600003480911255,
+        "tpp_threshold_500_unintended_diff_only": 0.003999993205070496
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.0009999573230743408,
+        "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
+        "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162,
+        "tpp_threshold_5_total_metric": -0.007750019431114197,
+        "tpp_threshold_5_intended_diff_only": 0.001999974250793457,
+        "tpp_threshold_5_unintended_diff_only": 0.009749993681907654,
+        "tpp_threshold_10_total_metric": 0.007749974727630615,
+        "tpp_threshold_10_intended_diff_only": 0.010999977588653564,
+        "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_20_total_metric": 0.005499944090843201,
+        "tpp_threshold_20_intended_diff_only": 0.007999956607818604,
+        "tpp_threshold_20_unintended_diff_only": 0.002500012516975403,
+        "tpp_threshold_50_total_metric": 0.013999953866004944,
+        "tpp_threshold_50_intended_diff_only": 0.0209999680519104,
+        "tpp_threshold_50_unintended_diff_only": 0.0070000141859054565,
+        "tpp_threshold_100_total_metric": 0.02374996244907379,
+        "tpp_threshold_100_intended_diff_only": 0.034999966621398926,
+        "tpp_threshold_100_unintended_diff_only": 0.011250004172325134,
+        "tpp_threshold_500_total_metric": 0.11574997007846832,
+        "tpp_threshold_500_intended_diff_only": 0.12699997425079346,
+        "tpp_threshold_500_unintended_diff_only": 0.011250004172325134
+      },
+      "3": {
+        "tpp_threshold_2_total_metric": -0.007500007748603821,
+        "tpp_threshold_2_intended_diff_only": -0.0040000081062316895,
+        "tpp_threshold_2_unintended_diff_only": 0.0034999996423721313,
+        "tpp_threshold_5_total_metric": 0.0004999935626983643,
+        "tpp_threshold_5_intended_diff_only": 0.0009999871253967285,
+        "tpp_threshold_5_unintended_diff_only": 0.0004999935626983643,
+        "tpp_threshold_10_total_metric": 0.012000009417533875,
+        "tpp_threshold_10_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_10_unintended_diff_only": 0.001999989151954651,
+        "tpp_threshold_20_total_metric": 0.002749994397163391,
+        "tpp_threshold_20_intended_diff_only": 0.009000003337860107,
+        "tpp_threshold_20_unintended_diff_only": 0.006250008940696716,
+        "tpp_threshold_50_total_metric": 0.016749992966651917,
+        "tpp_threshold_50_intended_diff_only": 0.018999993801116943,
+        "tpp_threshold_50_unintended_diff_only": 0.002250000834465027,
+        "tpp_threshold_100_total_metric": 0.040249988436698914,
+        "tpp_threshold_100_intended_diff_only": 0.046999990940093994,
+        "tpp_threshold_100_unintended_diff_only": 0.006750002503395081,
+        "tpp_threshold_500_total_metric": 0.12049998342990875,
+        "tpp_threshold_500_intended_diff_only": 0.1340000033378601,
+        "tpp_threshold_500_unintended_diff_only": 0.013500019907951355
+      },
+      "5": {
+        "tpp_threshold_2_total_metric": -0.0054999589920043945,
+        "tpp_threshold_2_intended_diff_only": -0.001999974250793457,
+        "tpp_threshold_2_unintended_diff_only": 0.0034999847412109375,
+        "tpp_threshold_5_total_metric": -0.0027499794960021973,
+        "tpp_threshold_5_intended_diff_only": 0.003000020980834961,
+        "tpp_threshold_5_unintended_diff_only": 0.005750000476837158,
+        "tpp_threshold_10_total_metric": -0.007249996066093445,
+        "tpp_threshold_10_intended_diff_only": -0.0009999871253967285,
+        "tpp_threshold_10_unintended_diff_only": 0.006250008940696716,
+        "tpp_threshold_20_total_metric": 0.008249983191490173,
+        "tpp_threshold_20_intended_diff_only": 0.014999985694885254,
+        "tpp_threshold_20_unintended_diff_only": 0.006750002503395081,
+        "tpp_threshold_50_total_metric": 0.020749986171722412,
+        "tpp_threshold_50_intended_diff_only": 0.02899998426437378,
+        "tpp_threshold_50_unintended_diff_only": 0.008249998092651367,
+        "tpp_threshold_100_total_metric": 0.03675001859664917,
+        "tpp_threshold_100_intended_diff_only": 0.04900002479553223,
+        "tpp_threshold_100_unintended_diff_only": 0.012250006198883057,
+        "tpp_threshold_500_total_metric": 0.11750003695487976,
+        "tpp_threshold_500_intended_diff_only": 0.13600003719329834,
+        "tpp_threshold_500_unintended_diff_only": 0.01850000023841858
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.016999974846839905,
+        "tpp_threshold_2_intended_diff_only": 0.019999980926513672,
+        "tpp_threshold_2_unintended_diff_only": 0.003000006079673767,
+        "tpp_threshold_5_total_metric": 0.01124998927116394,
+        "tpp_threshold_5_intended_diff_only": 0.009999990463256836,
+        "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045,
+        "tpp_threshold_10_total_metric": 0.03825001418590546,
+        "tpp_threshold_10_intended_diff_only": 0.04100000858306885,
+        "tpp_threshold_10_unintended_diff_only": 0.002749994397163391,
+        "tpp_threshold_20_total_metric": 0.04175001382827759,
+        "tpp_threshold_20_intended_diff_only": 0.04500001668930054,
+        "tpp_threshold_20_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_50_total_metric": 0.06299997866153717,
+        "tpp_threshold_50_intended_diff_only": 0.07099997997283936,
+        "tpp_threshold_50_unintended_diff_only": 0.008000001311302185,
+        "tpp_threshold_100_total_metric": 0.08100000023841858,
+        "tpp_threshold_100_intended_diff_only": 0.08899998664855957,
+        "tpp_threshold_100_unintended_diff_only": 0.007999986410140991,
+        "tpp_threshold_500_total_metric": 0.247250035405159,
+        "tpp_threshold_500_intended_diff_only": 0.2560000419616699,
+        "tpp_threshold_500_unintended_diff_only": 0.008750006556510925
+      }
+    }
+  }
+}

old_relu_eval_results/tpp/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_5_custom_sae_eval_results.json

@@ -0,0 +1,414 @@
+{
+  "eval_type_id": "tpp",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": false,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "9cc117bf-8880-4c1e-be3d-07555801ce77",
+  "datetime_epoch_millis": 1738808138146,
+  "eval_result_metrics": {
+    "tpp_metrics": {
+      "tpp_threshold_2_total_metric": 0.0010750010609626769,
+      "tpp_threshold_2_intended_diff_only": 0.0034000039100646976,
+      "tpp_threshold_2_unintended_diff_only": 0.00232500284910202,
+      "tpp_threshold_5_total_metric": 0.0006999984383583069,
+      "tpp_threshold_5_intended_diff_only": 0.0031000018119812013,
+      "tpp_threshold_5_unintended_diff_only": 0.0024000033736228943,
+      "tpp_threshold_10_total_metric": 0.005324994027614593,
+      "tpp_threshold_10_intended_diff_only": 0.008099997043609619,
+      "tpp_threshold_10_unintended_diff_only": 0.0027750030159950255,
+      "tpp_threshold_20_total_metric": 0.009074991941452027,
+      "tpp_threshold_20_intended_diff_only": 0.01199999451637268,
+      "tpp_threshold_20_unintended_diff_only": 0.0029250025749206544,
+      "tpp_threshold_50_total_metric": 0.019300003349781037,
+      "tpp_threshold_50_intended_diff_only": 0.02290000319480896,
+      "tpp_threshold_50_unintended_diff_only": 0.003599999845027924,
+      "tpp_threshold_100_total_metric": 0.029224996268749238,
+      "tpp_threshold_100_intended_diff_only": 0.03529999852180481,
+      "tpp_threshold_100_unintended_diff_only": 0.0060750022530555725,
+      "tpp_threshold_500_total_metric": 0.11080000549554825,
+      "tpp_threshold_500_intended_diff_only": 0.11790000796318054,
+      "tpp_threshold_500_unintended_diff_only": 0.007100002467632293
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
+      "tpp_threshold_2_total_metric": 0.002349993586540222,
+      "tpp_threshold_2_intended_diff_only": 0.0039999961853027345,
+      "tpp_threshold_2_unintended_diff_only": 0.0016500025987625123,
+      "tpp_threshold_5_total_metric": 0.003099992871284485,
+      "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
+      "tpp_threshold_5_unintended_diff_only": 0.001900002360343933,
+      "tpp_threshold_10_total_metric": 0.006099992990493774,
+      "tpp_threshold_10_intended_diff_only": 0.007799994945526123,
+      "tpp_threshold_10_unintended_diff_only": 0.0017000019550323486,
+      "tpp_threshold_20_total_metric": 0.011999988555908203,
+      "tpp_threshold_20_intended_diff_only": 0.013399994373321534,
+      "tpp_threshold_20_unintended_diff_only": 0.0014000058174133301,
+      "tpp_threshold_50_total_metric": 0.02170000672340393,
+      "tpp_threshold_50_intended_diff_only": 0.02380000352859497,
+      "tpp_threshold_50_unintended_diff_only": 0.00209999680519104,
+      "tpp_threshold_100_total_metric": 0.03549998998641968,
+      "tpp_threshold_100_intended_diff_only": 0.03879998922348023,
+      "tpp_threshold_100_unintended_diff_only": 0.003299999237060547,
+      "tpp_threshold_500_total_metric": 0.1364999920129776,
+      "tpp_threshold_500_intended_diff_only": 0.14199999570846558,
+      "tpp_threshold_500_unintended_diff_only": 0.005500003695487976
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
+      "tpp_threshold_2_total_metric": -0.00019999146461486817,
+      "tpp_threshold_2_intended_diff_only": 0.0028000116348266602,
+      "tpp_threshold_2_unintended_diff_only": 0.003000003099441528,
+      "tpp_threshold_5_total_metric": -0.0016999959945678711,
+      "tpp_threshold_5_intended_diff_only": 0.0012000083923339843,
+      "tpp_threshold_5_unintended_diff_only": 0.0029000043869018555,
+      "tpp_threshold_10_total_metric": 0.004549995064735413,
+      "tpp_threshold_10_intended_diff_only": 0.008399999141693116,
+      "tpp_threshold_10_unintended_diff_only": 0.0038500040769577025,
+      "tpp_threshold_20_total_metric": 0.00614999532699585,
+      "tpp_threshold_20_intended_diff_only": 0.010599994659423828,
+      "tpp_threshold_20_unintended_diff_only": 0.004449999332427979,
+      "tpp_threshold_50_total_metric": 0.016899999976158143,
+      "tpp_threshold_50_intended_diff_only": 0.02200000286102295,
+      "tpp_threshold_50_unintended_diff_only": 0.0051000028848648075,
+      "tpp_threshold_100_total_metric": 0.022950002551078798,
+      "tpp_threshold_100_intended_diff_only": 0.03180000782012939,
+      "tpp_threshold_100_unintended_diff_only": 0.008850005269050599,
+      "tpp_threshold_500_total_metric": 0.08510001897811889,
+      "tpp_threshold_500_intended_diff_only": 0.0938000202178955,
+      "tpp_threshold_500_unintended_diff_only": 0.00870000123977661
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_5",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1": {
+      "0": {
+        "tpp_threshold_2_total_metric": 0.005500033497810364,
+        "tpp_threshold_2_intended_diff_only": 0.00700002908706665,
+        "tpp_threshold_2_unintended_diff_only": 0.0014999955892562866,
+        "tpp_threshold_5_total_metric": 0.010000035166740417,
+        "tpp_threshold_5_intended_diff_only": 0.01100003719329834,
+        "tpp_threshold_5_unintended_diff_only": 0.0010000020265579224,
+        "tpp_threshold_10_total_metric": 0.002500012516975403,
+        "tpp_threshold_10_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_10_unintended_diff_only": 0.0014999955892562866,
+        "tpp_threshold_20_total_metric": 0.008999988436698914,
+        "tpp_threshold_20_intended_diff_only": 0.009999990463256836,
+        "tpp_threshold_20_unintended_diff_only": 0.0010000020265579224,
+        "tpp_threshold_50_total_metric": 0.016499996185302734,
+        "tpp_threshold_50_intended_diff_only": 0.018999993801116943,
+        "tpp_threshold_50_unintended_diff_only": 0.002499997615814209,
+        "tpp_threshold_100_total_metric": 0.02399998903274536,
+        "tpp_threshold_100_intended_diff_only": 0.02799999713897705,
+        "tpp_threshold_100_unintended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_500_total_metric": 0.12150004506111145,
+        "tpp_threshold_500_intended_diff_only": 0.12200003862380981,
+        "tpp_threshold_500_unintended_diff_only": 0.0004999935626983643
+      },
+      "1": {
+        "tpp_threshold_2_total_metric": 0.0032499581575393677,
+        "tpp_threshold_2_intended_diff_only": 0.0029999613761901855,
+        "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213,
+        "tpp_threshold_5_total_metric": -0.0025000572204589844,
+        "tpp_threshold_5_intended_diff_only": -0.001000046730041504,
+        "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805,
+        "tpp_threshold_10_total_metric": 0.002499982714653015,
+        "tpp_threshold_10_intended_diff_only": 0.0009999871253967285,
+        "tpp_threshold_10_unintended_diff_only": -0.0014999955892562866,
+        "tpp_threshold_20_total_metric": 0.0029999911785125732,
+        "tpp_threshold_20_intended_diff_only": 0.004999995231628418,
+        "tpp_threshold_20_unintended_diff_only": 0.0020000040531158447,
+        "tpp_threshold_50_total_metric": 0.02799999713897705,
+        "tpp_threshold_50_intended_diff_only": 0.03200000524520874,
+        "tpp_threshold_50_unintended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_100_total_metric": 0.039999961853027344,
+        "tpp_threshold_100_intended_diff_only": 0.04499995708465576,
+        "tpp_threshold_100_unintended_diff_only": 0.004999995231628418,
+        "tpp_threshold_500_total_metric": 0.11849997937679291,
+        "tpp_threshold_500_intended_diff_only": 0.12599998712539673,
+        "tpp_threshold_500_unintended_diff_only": 0.007500007748603821
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": -0.000500023365020752,
+        "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
+        "tpp_threshold_2_unintended_diff_only": 0.002499997615814209,
+        "tpp_threshold_5_total_metric": 0.002249971032142639,
+        "tpp_threshold_5_intended_diff_only": 0.0059999823570251465,
+        "tpp_threshold_5_unintended_diff_only": 0.0037500113248825073,
+        "tpp_threshold_10_total_metric": 0.01600000262260437,
+        "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447,
+        "tpp_threshold_20_total_metric": 0.02374999225139618,
+        "tpp_threshold_20_intended_diff_only": 0.02399998903274536,
+        "tpp_threshold_20_unintended_diff_only": 0.00024999678134918213,
+        "tpp_threshold_50_total_metric": 0.01800002157688141,
+        "tpp_threshold_50_intended_diff_only": 0.017000019550323486,
+        "tpp_threshold_50_unintended_diff_only": -0.0010000020265579224,
+        "tpp_threshold_100_total_metric": 0.01874999701976776,
+        "tpp_threshold_100_intended_diff_only": 0.018999993801116943,
+        "tpp_threshold_100_unintended_diff_only": 0.00024999678134918213,
+        "tpp_threshold_500_total_metric": 0.0807499885559082,
+        "tpp_threshold_500_intended_diff_only": 0.07999998331069946,
+        "tpp_threshold_500_unintended_diff_only": -0.0007500052452087402
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.0017500072717666626,
+        "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
+        "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983,
+        "tpp_threshold_5_total_metric": 0.0022500455379486084,
+        "tpp_threshold_5_intended_diff_only": 0.0020000338554382324,
+        "tpp_threshold_5_unintended_diff_only": -0.000250011682510376,
+        "tpp_threshold_10_total_metric": 0.0032499730587005615,
+        "tpp_threshold_10_intended_diff_only": 0.0059999823570251465,
+        "tpp_threshold_10_unintended_diff_only": 0.002750009298324585,
+        "tpp_threshold_20_total_metric": 0.006749972701072693,
+        "tpp_threshold_20_intended_diff_only": 0.0059999823570251465,
+        "tpp_threshold_20_unintended_diff_only": -0.0007499903440475464,
+        "tpp_threshold_50_total_metric": 0.009000048041343689,
+        "tpp_threshold_50_intended_diff_only": 0.01100003719329834,
+        "tpp_threshold_50_unintended_diff_only": 0.001999989151954651,
+        "tpp_threshold_100_total_metric": 0.016749978065490723,
+        "tpp_threshold_100_intended_diff_only": 0.019999980926513672,
+        "tpp_threshold_100_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_500_total_metric": 0.16824999451637268,
+        "tpp_threshold_500_intended_diff_only": 0.1809999942779541,
+        "tpp_threshold_500_unintended_diff_only": 0.012749999761581421
+      },
+      "9": {
+        "tpp_threshold_2_total_metric": 0.0017499923706054688,
+        "tpp_threshold_2_intended_diff_only": 0.004999995231628418,
+        "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_5_total_metric": 0.0034999698400497437,
+        "tpp_threshold_5_intended_diff_only": 0.006999969482421875,
+        "tpp_threshold_5_unintended_diff_only": 0.0034999996423721313,
+        "tpp_threshold_10_total_metric": 0.0062499940395355225,
+        "tpp_threshold_10_intended_diff_only": 0.009999990463256836,
+        "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_20_total_metric": 0.017499998211860657,
+        "tpp_threshold_20_intended_diff_only": 0.022000014781951904,
+        "tpp_threshold_20_unintended_diff_only": 0.0045000165700912476,
+        "tpp_threshold_50_total_metric": 0.03699997067451477,
+        "tpp_threshold_50_intended_diff_only": 0.039999961853027344,
+        "tpp_threshold_50_unintended_diff_only": 0.0029999911785125732,
+        "tpp_threshold_100_total_metric": 0.0780000239610672,
+        "tpp_threshold_100_intended_diff_only": 0.0820000171661377,
+        "tpp_threshold_100_unintended_diff_only": 0.003999993205070496,
+        "tpp_threshold_500_total_metric": 0.19349995255470276,
+        "tpp_threshold_500_intended_diff_only": 0.20099997520446777,
+        "tpp_threshold_500_unintended_diff_only": 0.007500022649765015
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5": {
+      "1": {
+        "tpp_threshold_2_total_metric": 0.01175004243850708,
+        "tpp_threshold_2_intended_diff_only": 0.01500004529953003,
+        "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_5_total_metric": 0.006250008940696716,
+        "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
+        "tpp_threshold_5_unintended_diff_only": 0.0017500072717666626,
+        "tpp_threshold_10_total_metric": 0.003000006079673767,
+        "tpp_threshold_10_intended_diff_only": 0.009000003337860107,
+        "tpp_threshold_10_unintended_diff_only": 0.00599999725818634,
+        "tpp_threshold_20_total_metric": 0.003750041127204895,
+        "tpp_threshold_20_intended_diff_only": 0.00700002908706665,
+        "tpp_threshold_20_unintended_diff_only": 0.0032499879598617554,
+        "tpp_threshold_50_total_metric": 0.015000015497207642,
+        "tpp_threshold_50_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_50_unintended_diff_only": -0.0010000169277191162,
+        "tpp_threshold_100_total_metric": 0.00850003957748413,
+        "tpp_threshold_100_intended_diff_only": 0.01500004529953003,
+        "tpp_threshold_100_unintended_diff_only": 0.0065000057220458984,
+        "tpp_threshold_500_total_metric": 0.033750057220458984,
+        "tpp_threshold_500_intended_diff_only": 0.03800004720687866,
+        "tpp_threshold_500_unintended_diff_only": 0.004249989986419678
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.0012499690055847168,
+        "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
+        "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402,
+        "tpp_threshold_5_total_metric": -0.009000003337860107,
+        "tpp_threshold_5_intended_diff_only": 0.0,
+        "tpp_threshold_5_unintended_diff_only": 0.009000003337860107,
+        "tpp_threshold_10_total_metric": 0.004499971866607666,
+        "tpp_threshold_10_intended_diff_only": 0.006999969482421875,
+        "tpp_threshold_10_unintended_diff_only": 0.002499997615814209,
+        "tpp_threshold_20_total_metric": -0.0012499988079071045,
+        "tpp_threshold_20_intended_diff_only": 0.0,
+        "tpp_threshold_20_unintended_diff_only": 0.0012499988079071045,
+        "tpp_threshold_50_total_metric": 0.00349995493888855,
+        "tpp_threshold_50_intended_diff_only": 0.010999977588653564,
+        "tpp_threshold_50_unintended_diff_only": 0.007500022649765015,
+        "tpp_threshold_100_total_metric": 0.00899997353553772,
+        "tpp_threshold_100_intended_diff_only": 0.019999980926513672,
+        "tpp_threshold_100_unintended_diff_only": 0.011000007390975952,
+        "tpp_threshold_500_total_metric": 0.07375001907348633,
+        "tpp_threshold_500_intended_diff_only": 0.0820000171661377,
+        "tpp_threshold_500_unintended_diff_only": 0.008249998092651367
+      },
+      "3": {
+        "tpp_threshold_2_total_metric": -0.009000003337860107,
+        "tpp_threshold_2_intended_diff_only": -0.004999995231628418,
+        "tpp_threshold_2_unintended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_5_total_metric": -0.0052499920129776,
+        "tpp_threshold_5_intended_diff_only": -0.0059999823570251465,
+        "tpp_threshold_5_unintended_diff_only": -0.0007499903440475464,
+        "tpp_threshold_10_total_metric": 0.007499992847442627,
+        "tpp_threshold_10_intended_diff_only": 0.009000003337860107,
+        "tpp_threshold_10_unintended_diff_only": 0.0015000104904174805,
+        "tpp_threshold_20_total_metric": -0.004000037908554077,
+        "tpp_threshold_20_intended_diff_only": 0.001999974250793457,
+        "tpp_threshold_20_unintended_diff_only": 0.006000012159347534,
+        "tpp_threshold_50_total_metric": 0.010999992489814758,
+        "tpp_threshold_50_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_50_unintended_diff_only": 0.003000006079673767,
+        "tpp_threshold_100_total_metric": 0.030500009655952454,
+        "tpp_threshold_100_intended_diff_only": 0.03600001335144043,
+        "tpp_threshold_100_unintended_diff_only": 0.005500003695487976,
+        "tpp_threshold_500_total_metric": 0.07175000011920929,
+        "tpp_threshold_500_intended_diff_only": 0.08300000429153442,
+        "tpp_threshold_500_unintended_diff_only": 0.011250004172325134
+      },
+      "5": {
+        "tpp_threshold_2_total_metric": -0.009249985218048096,
+        "tpp_threshold_2_intended_diff_only": -0.0059999823570251465,
+        "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_5_total_metric": -0.006749987602233887,
+        "tpp_threshold_5_intended_diff_only": -0.0009999871253967285,
+        "tpp_threshold_5_unintended_diff_only": 0.005750000476837158,
+        "tpp_threshold_10_total_metric": -0.012249991297721863,
+        "tpp_threshold_10_intended_diff_only": -0.006999969482421875,
+        "tpp_threshold_10_unintended_diff_only": 0.005250021815299988,
+        "tpp_threshold_20_total_metric": -0.005250006914138794,
+        "tpp_threshold_20_intended_diff_only": 0.0009999871253967285,
+        "tpp_threshold_20_unintended_diff_only": 0.0062499940395355225,
+        "tpp_threshold_50_total_metric": 0.005750015377998352,
+        "tpp_threshold_50_intended_diff_only": 0.013000011444091797,
+        "tpp_threshold_50_unintended_diff_only": 0.007249996066093445,
+        "tpp_threshold_100_total_metric": 0.009499996900558472,
+        "tpp_threshold_100_intended_diff_only": 0.023000001907348633,
+        "tpp_threshold_100_unintended_diff_only": 0.013500005006790161,
+        "tpp_threshold_500_total_metric": 0.06675000488758087,
+        "tpp_threshold_500_intended_diff_only": 0.078000009059906,
+        "tpp_threshold_500_unintended_diff_only": 0.011250004172325134
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.004250019788742065,
+        "tpp_threshold_2_intended_diff_only": 0.008000016212463379,
+        "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_5_total_metric": 0.0062499940395355225,
+        "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
+        "tpp_threshold_5_unintended_diff_only": -0.0012499988079071045,
+        "tpp_threshold_10_total_metric": 0.019999995827674866,
+        "tpp_threshold_10_intended_diff_only": 0.02399998903274536,
+        "tpp_threshold_10_unintended_diff_only": 0.003999993205070496,
+        "tpp_threshold_20_total_metric": 0.03749997913837433,
+        "tpp_threshold_20_intended_diff_only": 0.042999982833862305,
+        "tpp_threshold_20_unintended_diff_only": 0.005500003695487976,
+        "tpp_threshold_50_total_metric": 0.04925002157688141,
+        "tpp_threshold_50_intended_diff_only": 0.058000028133392334,
+        "tpp_threshold_50_unintended_diff_only": 0.008750006556510925,
+        "tpp_threshold_100_total_metric": 0.057249993085861206,
+        "tpp_threshold_100_intended_diff_only": 0.06499999761581421,
+        "tpp_threshold_100_unintended_diff_only": 0.007750004529953003,
+        "tpp_threshold_500_total_metric": 0.179500013589859,
+        "tpp_threshold_500_intended_diff_only": 0.18800002336502075,
+        "tpp_threshold_500_unintended_diff_only": 0.008500009775161743
+      }
+    }
+  }
+}

old_relu_eval_results/unlearning/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_0_custom_sae_eval_results.json

@@ -0,0 +1,74 @@
+{
+  "eval_type_id": "unlearning",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "wmdp-bio",
+      "high_school_us_history",
+      "college_computer_science",
+      "high_school_geography",
+      "human_aging"
+    ],
+    "intervention_method": "clamp_feature_activation",
+    "retain_thresholds": [
+      0.001,
+      0.01
+    ],
+    "n_features_list": [
+      10,
+      20
+    ],
+    "multipliers": [
+      25,
+      50,
+      100,
+      200
+    ],
+    "dataset_size": 1024,
+    "seq_len": 1024,
+    "n_batch_loss_added": 50,
+    "target_metric": "correct",
+    "save_metrics": true,
+    "model_name": "gemma-2-2b-it",
+    "llm_batch_size": 4,
+    "llm_dtype": "bfloat16"
+  },
+  "eval_id": "27152fa7-494d-44cf-ac35-e2e33052a1b4",
+  "datetime_epoch_millis": 1738815248274,
+  "eval_result_metrics": {
+    "unlearning": {
+      "unlearning_score": 0.007504701614379883
+    }
+  },
+  "eval_result_details": [],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_0",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/unlearning/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_1_custom_sae_eval_results.json

@@ -0,0 +1,74 @@
+{
+  "eval_type_id": "unlearning",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "wmdp-bio",
+      "high_school_us_history",
+      "college_computer_science",
+      "high_school_geography",
+      "human_aging"
+    ],
+    "intervention_method": "clamp_feature_activation",
+    "retain_thresholds": [
+      0.001,
+      0.01
+    ],
+    "n_features_list": [
+      10,
+      20
+    ],
+    "multipliers": [
+      25,
+      50,
+      100,
+      200
+    ],
+    "dataset_size": 1024,
+    "seq_len": 1024,
+    "n_batch_loss_added": 50,
+    "target_metric": "correct",
+    "save_metrics": true,
+    "model_name": "gemma-2-2b-it",
+    "llm_batch_size": 4,
+    "llm_dtype": "bfloat16"
+  },
+  "eval_id": "b6ab776e-c77a-46bb-9b89-5bd9905335f9",
+  "datetime_epoch_millis": 1738815974660,
+  "eval_result_metrics": {
+    "unlearning": {
+      "unlearning_score": 0.015009403228759766
+    }
+  },
+  "eval_result_details": [],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_1",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/unlearning/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_2_custom_sae_eval_results.json

@@ -0,0 +1,74 @@
+{
+  "eval_type_id": "unlearning",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "wmdp-bio",
+      "high_school_us_history",
+      "college_computer_science",
+      "high_school_geography",
+      "human_aging"
+    ],
+    "intervention_method": "clamp_feature_activation",
+    "retain_thresholds": [
+      0.001,
+      0.01
+    ],
+    "n_features_list": [
+      10,
+      20
+    ],
+    "multipliers": [
+      25,
+      50,
+      100,
+      200
+    ],
+    "dataset_size": 1024,
+    "seq_len": 1024,
+    "n_batch_loss_added": 50,
+    "target_metric": "correct",
+    "save_metrics": true,
+    "model_name": "gemma-2-2b-it",
+    "llm_batch_size": 4,
+    "llm_dtype": "bfloat16"
+  },
+  "eval_id": "4e677628-eb8d-40f5-ade8-e25072775bf1",
+  "datetime_epoch_millis": 1738818108714,
+  "eval_result_metrics": {
+    "unlearning": {
+      "unlearning_score": 0.06378984451293945
+    }
+  },
+  "eval_result_details": [],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_2",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/unlearning/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_3_custom_sae_eval_results.json

@@ -0,0 +1,74 @@
+{
+  "eval_type_id": "unlearning",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "wmdp-bio",
+      "high_school_us_history",
+      "college_computer_science",
+      "high_school_geography",
+      "human_aging"
+    ],
+    "intervention_method": "clamp_feature_activation",
+    "retain_thresholds": [
+      0.001,
+      0.01
+    ],
+    "n_features_list": [
+      10,
+      20
+    ],
+    "multipliers": [
+      25,
+      50,
+      100,
+      200
+    ],
+    "dataset_size": 1024,
+    "seq_len": 1024,
+    "n_batch_loss_added": 50,
+    "target_metric": "correct",
+    "save_metrics": true,
+    "model_name": "gemma-2-2b-it",
+    "llm_batch_size": 4,
+    "llm_dtype": "bfloat16"
+  },
+  "eval_id": "9001c398-92b1-4daf-8d14-cd80c5147849",
+  "datetime_epoch_millis": 1738817408434,
+  "eval_result_metrics": {
+    "unlearning": {
+      "unlearning_score": 0.0863039493560791
+    }
+  },
+  "eval_result_details": [],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_3",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/unlearning/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_4_custom_sae_eval_results.json

@@ -0,0 +1,74 @@
+{
+  "eval_type_id": "unlearning",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "wmdp-bio",
+      "high_school_us_history",
+      "college_computer_science",
+      "high_school_geography",
+      "human_aging"
+    ],
+    "intervention_method": "clamp_feature_activation",
+    "retain_thresholds": [
+      0.001,
+      0.01
+    ],
+    "n_features_list": [
+      10,
+      20
+    ],
+    "multipliers": [
+      25,
+      50,
+      100,
+      200
+    ],
+    "dataset_size": 1024,
+    "seq_len": 1024,
+    "n_batch_loss_added": 50,
+    "target_metric": "correct",
+    "save_metrics": true,
+    "model_name": "gemma-2-2b-it",
+    "llm_batch_size": 4,
+    "llm_dtype": "bfloat16"
+  },
+  "eval_id": "68919444-b3fe-445b-b7d6-7eadd952f2a7",
+  "datetime_epoch_millis": 1738818814082,
+  "eval_result_metrics": {
+    "unlearning": {
+      "unlearning_score": 0.09943711757659912
+    }
+  },
+  "eval_result_details": [],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_4",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

old_relu_eval_results/unlearning/temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_5_custom_sae_eval_results.json

@@ -0,0 +1,74 @@
+{
+  "eval_type_id": "unlearning",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "wmdp-bio",
+      "high_school_us_history",
+      "college_computer_science",
+      "high_school_geography",
+      "human_aging"
+    ],
+    "intervention_method": "clamp_feature_activation",
+    "retain_thresholds": [
+      0.001,
+      0.01
+    ],
+    "n_features_list": [
+      10,
+      20
+    ],
+    "multipliers": [
+      25,
+      50,
+      100,
+      200
+    ],
+    "dataset_size": 1024,
+    "seq_len": 1024,
+    "n_batch_loss_added": 50,
+    "target_metric": "correct",
+    "save_metrics": true,
+    "model_name": "gemma-2-2b-it",
+    "llm_batch_size": 4,
+    "llm_dtype": "bfloat16"
+  },
+  "eval_id": "39440e8a-6927-4b07-b9d8-8bf75861a3b2",
+  "datetime_epoch_millis": 1738816689242,
+  "eval_result_metrics": {
+    "unlearning": {
+      "unlearning_score": 0.23452156782150269
+    }
+  },
+  "eval_result_details": [],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_old_relu_google_gemma-2-2b_standard_resid_post_layer_12_trainer_5",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "standard",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}