Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+random_seed_eval_results/autointerp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_1_custom_sae_eval_results.json filter=lfs diff=lfs merge=lfs -text
+random_seed_eval_results/autointerp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_0_custom_sae_eval_results.json filter=lfs diff=lfs merge=lfs -text
+random_seed_eval_results/autointerp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_3_custom_sae_eval_results.json filter=lfs diff=lfs merge=lfs -text
+random_seed_eval_results/autointerp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_4_custom_sae_eval_results.json filter=lfs diff=lfs merge=lfs -text
+random_seed_eval_results/autointerp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_2_custom_sae_eval_results.json filter=lfs diff=lfs merge=lfs -text

random_seed_eval_results/absorption/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_0_custom_sae_eval_results.json

@@ -0,0 +1,268 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50
+  },
+  "eval_id": "48f426fa-d13b-4265-8871-5585ec45578b",
+  "datetime_epoch_millis": 1738784829328,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.1714686461548928,
+      "mean_full_absorption_score": 0.15386796931592747,
+      "mean_num_split_features": 1.2307692307692308,
+      "std_dev_absorption_fraction_score": 0.15554948458633766,
+      "std_dev_full_absorption_score": 0.14785339675818804,
+      "std_dev_num_split_features": 0.5870395085642742
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.10821684724823694,
+      "full_absorption_rate": 0.05433932759968726,
+      "num_full_absorption": 139,
+      "num_probe_true_positives": 2558,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.030210348040788013,
+      "full_absorption_rate": 0.017879948914431672,
+      "num_full_absorption": 28,
+      "num_probe_true_positives": 1566,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.41961031215817146,
+      "full_absorption_rate": 0.3772369362920544,
+      "num_full_absorption": 1054,
+      "num_probe_true_positives": 2794,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.3467083150960684,
+      "full_absorption_rate": 0.232981220657277,
+      "num_full_absorption": 397,
+      "num_probe_true_positives": 1704,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.2815750993327208,
+      "full_absorption_rate": 0.2759493670886076,
+      "num_full_absorption": 436,
+      "num_probe_true_positives": 1580,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.15487083167478738,
+      "full_absorption_rate": 0.11884550084889643,
+      "num_full_absorption": 140,
+      "num_probe_true_positives": 1178,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.07421657717906203,
+      "full_absorption_rate": 0.07180851063829788,
+      "num_full_absorption": 81,
+      "num_probe_true_positives": 1128,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.05684645316227957,
+      "full_absorption_rate": 0.026449643947100712,
+      "num_full_absorption": 26,
+      "num_probe_true_positives": 983,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.36796796561960116,
+      "full_absorption_rate": 0.39185140802875973,
+      "num_full_absorption": 654,
+      "num_probe_true_positives": 1669,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.001114532296360666,
+      "full_absorption_rate": 0.01366742596810934,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 439,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.0005988111448050127,
+      "full_absorption_rate": 0.004310344827586207,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 696,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.18851606271745078,
+      "full_absorption_rate": 0.18158347676419967,
+      "num_full_absorption": 211,
+      "num_probe_true_positives": 1162,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.2570751753813015,
+      "full_absorption_rate": 0.1906318082788671,
+      "num_full_absorption": 350,
+      "num_probe_true_positives": 1836,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.06637348713907784,
+      "full_absorption_rate": 0.05115712545676005,
+      "num_full_absorption": 42,
+      "num_probe_true_positives": 821,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.18381166786624634,
+      "full_absorption_rate": 0.21673003802281368,
+      "num_full_absorption": 228,
+      "num_probe_true_positives": 1052,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.4767444460243892,
+      "full_absorption_rate": 0.4237362637362637,
+      "num_full_absorption": 964,
+      "num_probe_true_positives": 2275,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.014007514920590125,
+      "full_absorption_rate": 0.020512820512820513,
+      "num_full_absorption": 4,
+      "num_probe_true_positives": 195,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.3248090798755296,
+      "full_absorption_rate": 0.2704773129051267,
+      "num_full_absorption": 459,
+      "num_probe_true_positives": 1697,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.47029052751768835,
+      "full_absorption_rate": 0.4606123151014792,
+      "num_full_absorption": 1339,
+      "num_probe_true_positives": 2907,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.17209049881072833,
+      "full_absorption_rate": 0.09523809523809523,
+      "num_full_absorption": 160,
+      "num_probe_true_positives": 1680,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.282661389531355,
+      "full_absorption_rate": 0.33077905491698595,
+      "num_full_absorption": 259,
+      "num_probe_true_positives": 783,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.05199868006616804,
+      "full_absorption_rate": 0.05089058524173028,
+      "num_full_absorption": 40,
+      "num_probe_true_positives": 786,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.06104382806297723,
+      "full_absorption_rate": 0.08006279434850863,
+      "num_full_absorption": 51,
+      "num_probe_true_positives": 637,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.05422639180774973,
+      "full_absorption_rate": 0.019417475728155338,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 103,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.01259995735307954,
+      "full_absorption_rate": 0.011560693641618497,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 173,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.0,
+      "full_absorption_rate": 0.011857707509881422,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 253,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_0",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

random_seed_eval_results/absorption/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_1_custom_sae_eval_results.json

@@ -0,0 +1,268 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50
+  },
+  "eval_id": "65edad50-a831-4080-9ef3-0543dc2bdb6d",
+  "datetime_epoch_millis": 1738783917967,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.1711332913395652,
+      "mean_full_absorption_score": 0.15146027486674538,
+      "mean_num_split_features": 1.3076923076923077,
+      "std_dev_absorption_fraction_score": 0.16355798531094715,
+      "std_dev_full_absorption_score": 0.15005451741951137,
+      "std_dev_num_split_features": 0.7358929688062399
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.13034142230683501,
+      "full_absorption_rate": 0.0727130570758405,
+      "num_full_absorption": 186,
+      "num_probe_true_positives": 2558,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.03860032573759929,
+      "full_absorption_rate": 0.02234993614303959,
+      "num_full_absorption": 35,
+      "num_probe_true_positives": 1566,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.48111974773989397,
+      "full_absorption_rate": 0.4359341445955619,
+      "num_full_absorption": 1218,
+      "num_probe_true_positives": 2794,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.490319824411387,
+      "full_absorption_rate": 0.4025821596244131,
+      "num_full_absorption": 686,
+      "num_probe_true_positives": 1704,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.2562253140403955,
+      "full_absorption_rate": 0.24746835443037973,
+      "num_full_absorption": 391,
+      "num_probe_true_positives": 1580,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.13061784782524904,
+      "full_absorption_rate": 0.09083191850594227,
+      "num_full_absorption": 107,
+      "num_probe_true_positives": 1178,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.08564969312088162,
+      "full_absorption_rate": 0.08067375886524823,
+      "num_full_absorption": 91,
+      "num_probe_true_positives": 1128,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.05383191935060708,
+      "full_absorption_rate": 0.024415055951169887,
+      "num_full_absorption": 24,
+      "num_probe_true_positives": 983,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.30973405186275194,
+      "full_absorption_rate": 0.32594367884961056,
+      "num_full_absorption": 544,
+      "num_probe_true_positives": 1669,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.0012255374185012769,
+      "full_absorption_rate": 0.01366742596810934,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 439,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.0002946951428704943,
+      "full_absorption_rate": 0.005747126436781609,
+      "num_full_absorption": 4,
+      "num_probe_true_positives": 696,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.20008709482189874,
+      "full_absorption_rate": 0.18932874354561102,
+      "num_full_absorption": 220,
+      "num_probe_true_positives": 1162,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.12115143225674394,
+      "full_absorption_rate": 0.08387799564270153,
+      "num_full_absorption": 154,
+      "num_probe_true_positives": 1836,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.05617340818231512,
+      "full_absorption_rate": 0.040194884287454324,
+      "num_full_absorption": 33,
+      "num_probe_true_positives": 821,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.19799995405281123,
+      "full_absorption_rate": 0.21387832699619772,
+      "num_full_absorption": 225,
+      "num_probe_true_positives": 1052,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.42373552854634816,
+      "full_absorption_rate": 0.3665934065934066,
+      "num_full_absorption": 834,
+      "num_probe_true_positives": 2275,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.019513367217384434,
+      "full_absorption_rate": 0.020512820512820513,
+      "num_full_absorption": 4,
+      "num_probe_true_positives": 195,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.38678566067757764,
+      "full_absorption_rate": 0.3252799057159694,
+      "num_full_absorption": 552,
+      "num_probe_true_positives": 1697,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.4675735623770602,
+      "full_absorption_rate": 0.4196766425868593,
+      "num_full_absorption": 1220,
+      "num_probe_true_positives": 2907,
+      "num_split_features": 4
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.19115083064649022,
+      "full_absorption_rate": 0.11964285714285715,
+      "num_full_absorption": 201,
+      "num_probe_true_positives": 1680,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.24797733259678253,
+      "full_absorption_rate": 0.30140485312899107,
+      "num_full_absorption": 236,
+      "num_probe_true_positives": 783,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.045272538797872366,
+      "full_absorption_rate": 0.043256997455470736,
+      "num_full_absorption": 34,
+      "num_probe_true_positives": 786,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.06742398501652283,
+      "full_absorption_rate": 0.06279434850863422,
+      "num_full_absorption": 40,
+      "num_probe_true_positives": 637,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.016888002349278078,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 103,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.02889315611561085,
+      "full_absorption_rate": 0.017341040462427744,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 173,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.0008793422170269473,
+      "full_absorption_rate": 0.011857707509881422,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 253,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_1",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

random_seed_eval_results/absorption/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_2_custom_sae_eval_results.json

@@ -0,0 +1,268 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50
+  },
+  "eval_id": "6aeabb62-1ccd-4e13-acae-ec063ce1b514",
+  "datetime_epoch_millis": 1738785735688,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.1680556606743435,
+      "mean_full_absorption_score": 0.15191023322678118,
+      "mean_num_split_features": 1.4615384615384615,
+      "std_dev_absorption_fraction_score": 0.14863662312386292,
+      "std_dev_full_absorption_score": 0.13647390993364086,
+      "std_dev_num_split_features": 0.9891721480417544
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.13047466988525097,
+      "full_absorption_rate": 0.0766223612197029,
+      "num_full_absorption": 196,
+      "num_probe_true_positives": 2558,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.023848564485708528,
+      "full_absorption_rate": 0.017241379310344827,
+      "num_full_absorption": 27,
+      "num_probe_true_positives": 1566,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.44742713898495284,
+      "full_absorption_rate": 0.3890479599141016,
+      "num_full_absorption": 1087,
+      "num_probe_true_positives": 2794,
+      "num_split_features": 4
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.3708607264352453,
+      "full_absorption_rate": 0.2711267605633803,
+      "num_full_absorption": 462,
+      "num_probe_true_positives": 1704,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.32176505353839885,
+      "full_absorption_rate": 0.3158227848101266,
+      "num_full_absorption": 499,
+      "num_probe_true_positives": 1580,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.14714927842788375,
+      "full_absorption_rate": 0.11120543293718166,
+      "num_full_absorption": 131,
+      "num_probe_true_positives": 1178,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.08731746860539064,
+      "full_absorption_rate": 0.08156028368794327,
+      "num_full_absorption": 92,
+      "num_probe_true_positives": 1128,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.055969492596627826,
+      "full_absorption_rate": 0.03153611393692777,
+      "num_full_absorption": 31,
+      "num_probe_true_positives": 983,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.2785833366797728,
+      "full_absorption_rate": 0.2995805871779509,
+      "num_full_absorption": 500,
+      "num_probe_true_positives": 1669,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.0017460357133270942,
+      "full_absorption_rate": 0.01366742596810934,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 439,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.0007196477033161077,
+      "full_absorption_rate": 0.005747126436781609,
+      "num_full_absorption": 4,
+      "num_probe_true_positives": 696,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.13922218898438524,
+      "full_absorption_rate": 0.1333907056798623,
+      "num_full_absorption": 155,
+      "num_probe_true_positives": 1162,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.18222620640573445,
+      "full_absorption_rate": 0.12418300653594772,
+      "num_full_absorption": 228,
+      "num_probe_true_positives": 1836,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.06866407604449835,
+      "full_absorption_rate": 0.05481120584652863,
+      "num_full_absorption": 45,
+      "num_probe_true_positives": 821,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.2887489800731137,
+      "full_absorption_rate": 0.3612167300380228,
+      "num_full_absorption": 380,
+      "num_probe_true_positives": 1052,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.44873846584887894,
+      "full_absorption_rate": 0.378021978021978,
+      "num_full_absorption": 860,
+      "num_probe_true_positives": 2275,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.041601474064769904,
+      "full_absorption_rate": 0.05641025641025641,
+      "num_full_absorption": 11,
+      "num_probe_true_positives": 195,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.3102650866902534,
+      "full_absorption_rate": 0.25987035945786685,
+      "num_full_absorption": 441,
+      "num_probe_true_positives": 1697,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.4414031454169644,
+      "full_absorption_rate": 0.4004127966976264,
+      "num_full_absorption": 1164,
+      "num_probe_true_positives": 2907,
+      "num_split_features": 5
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.2128788270041167,
+      "full_absorption_rate": 0.14345238095238094,
+      "num_full_absorption": 241,
+      "num_probe_true_positives": 1680,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.15207178611487968,
+      "full_absorption_rate": 0.22988505747126436,
+      "num_full_absorption": 180,
+      "num_probe_true_positives": 783,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.02597114917923278,
+      "full_absorption_rate": 0.03307888040712468,
+      "num_full_absorption": 26,
+      "num_probe_true_positives": 786,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.08939131791526586,
+      "full_absorption_rate": 0.08791208791208792,
+      "num_full_absorption": 56,
+      "num_probe_true_positives": 637,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.0674159775843633,
+      "full_absorption_rate": 0.019417475728155338,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 103,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.034987083150599306,
+      "full_absorption_rate": 0.03468208092485549,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 173,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.0,
+      "full_absorption_rate": 0.019762845849802372,
+      "num_full_absorption": 5,
+      "num_probe_true_positives": 253,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_2",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

random_seed_eval_results/absorption/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_3_custom_sae_eval_results.json

@@ -0,0 +1,268 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50
+  },
+  "eval_id": "a615335e-36f1-40d4-80c8-a632dba17d19",
+  "datetime_epoch_millis": 1738786653640,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.17538157541809793,
+      "mean_full_absorption_score": 0.1514468976607655,
+      "mean_num_split_features": 1.2692307692307692,
+      "std_dev_absorption_fraction_score": 0.16584926866768673,
+      "std_dev_full_absorption_score": 0.15332607817709293,
+      "std_dev_num_split_features": 0.6667948594698258
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.1843346778809071,
+      "full_absorption_rate": 0.0965598123534011,
+      "num_full_absorption": 247,
+      "num_probe_true_positives": 2558,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.027441769972842704,
+      "full_absorption_rate": 0.01532567049808429,
+      "num_full_absorption": 24,
+      "num_probe_true_positives": 1566,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.5391824266226292,
+      "full_absorption_rate": 0.4874731567644954,
+      "num_full_absorption": 1362,
+      "num_probe_true_positives": 2794,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.3774188362026092,
+      "full_absorption_rate": 0.2494131455399061,
+      "num_full_absorption": 425,
+      "num_probe_true_positives": 1704,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.33716358784198536,
+      "full_absorption_rate": 0.3120253164556962,
+      "num_full_absorption": 493,
+      "num_probe_true_positives": 1580,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.1350976074162719,
+      "full_absorption_rate": 0.100169779286927,
+      "num_full_absorption": 118,
+      "num_probe_true_positives": 1178,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.08468221916767832,
+      "full_absorption_rate": 0.0797872340425532,
+      "num_full_absorption": 90,
+      "num_probe_true_positives": 1128,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.0547531455889232,
+      "full_absorption_rate": 0.021363173957273652,
+      "num_full_absorption": 21,
+      "num_probe_true_positives": 983,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.3798311805573998,
+      "full_absorption_rate": 0.3996405032953865,
+      "num_full_absorption": 667,
+      "num_probe_true_positives": 1669,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.0035791146766769285,
+      "full_absorption_rate": 0.01366742596810934,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 439,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.0033612951454096854,
+      "full_absorption_rate": 0.007183908045977011,
+      "num_full_absorption": 5,
+      "num_probe_true_positives": 696,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.21961957691643827,
+      "full_absorption_rate": 0.1919104991394148,
+      "num_full_absorption": 223,
+      "num_probe_true_positives": 1162,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.08392659102380813,
+      "full_absorption_rate": 0.04847494553376906,
+      "num_full_absorption": 89,
+      "num_probe_true_positives": 1836,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.08025181361638342,
+      "full_absorption_rate": 0.06090133982947625,
+      "num_full_absorption": 50,
+      "num_probe_true_positives": 821,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.16317207769449307,
+      "full_absorption_rate": 0.17680608365019013,
+      "num_full_absorption": 186,
+      "num_probe_true_positives": 1052,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.5201401741517296,
+      "full_absorption_rate": 0.4320879120879121,
+      "num_full_absorption": 983,
+      "num_probe_true_positives": 2275,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.033809387805282545,
+      "full_absorption_rate": 0.046153846153846156,
+      "num_full_absorption": 9,
+      "num_probe_true_positives": 195,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.265286753988254,
+      "full_absorption_rate": 0.2139068945197407,
+      "num_full_absorption": 363,
+      "num_probe_true_positives": 1697,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.4506998861290704,
+      "full_absorption_rate": 0.4213966288269694,
+      "num_full_absorption": 1225,
+      "num_probe_true_positives": 2907,
+      "num_split_features": 4
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.20785809991779755,
+      "full_absorption_rate": 0.12083333333333333,
+      "num_full_absorption": 203,
+      "num_probe_true_positives": 1680,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.21319969783745676,
+      "full_absorption_rate": 0.30140485312899107,
+      "num_full_absorption": 236,
+      "num_probe_true_positives": 783,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.04640054552065163,
+      "full_absorption_rate": 0.03689567430025445,
+      "num_full_absorption": 29,
+      "num_probe_true_positives": 786,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.08355275384554323,
+      "full_absorption_rate": 0.0847723704866562,
+      "num_full_absorption": 54,
+      "num_probe_true_positives": 637,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.03616653815832325,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 103,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.018065386940751115,
+      "full_absorption_rate": 0.011560693641618497,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 173,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.010925816251230009,
+      "full_absorption_rate": 0.007905138339920948,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 253,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_3",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

random_seed_eval_results/absorption/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_4_custom_sae_eval_results.json

@@ -0,0 +1,268 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50
+  },
+  "eval_id": "4ab555a5-5c14-4096-985d-fbfcd95bad5d",
+  "datetime_epoch_millis": 1738787577551,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.16402077829695136,
+      "mean_full_absorption_score": 0.1455081445066387,
+      "mean_num_split_features": 1.2307692307692308,
+      "std_dev_absorption_fraction_score": 0.15788027063450677,
+      "std_dev_full_absorption_score": 0.1410482155509194,
+      "std_dev_num_split_features": 0.5870395085642742
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.12991450124362738,
+      "full_absorption_rate": 0.07584050039093042,
+      "num_full_absorption": 194,
+      "num_probe_true_positives": 2558,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.023356857769147497,
+      "full_absorption_rate": 0.01277139208173691,
+      "num_full_absorption": 20,
+      "num_probe_true_positives": 1566,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.42441637423084033,
+      "full_absorption_rate": 0.36435218324982105,
+      "num_full_absorption": 1018,
+      "num_probe_true_positives": 2794,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.3983830197774112,
+      "full_absorption_rate": 0.27582159624413144,
+      "num_full_absorption": 470,
+      "num_probe_true_positives": 1704,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.29735930296831414,
+      "full_absorption_rate": 0.28417721518987343,
+      "num_full_absorption": 449,
+      "num_probe_true_positives": 1580,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.14754169713635484,
+      "full_absorption_rate": 0.10696095076400679,
+      "num_full_absorption": 126,
+      "num_probe_true_positives": 1178,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.09041740227134817,
+      "full_absorption_rate": 0.09042553191489362,
+      "num_full_absorption": 102,
+      "num_probe_true_positives": 1128,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.043464844239960984,
+      "full_absorption_rate": 0.018311291963377416,
+      "num_full_absorption": 18,
+      "num_probe_true_positives": 983,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.28105418653291725,
+      "full_absorption_rate": 0.28819652486518876,
+      "num_full_absorption": 481,
+      "num_probe_true_positives": 1669,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.0011799746030814323,
+      "full_absorption_rate": 0.011389521640091117,
+      "num_full_absorption": 5,
+      "num_probe_true_positives": 439,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.0011871688335180757,
+      "full_absorption_rate": 0.010057471264367816,
+      "num_full_absorption": 7,
+      "num_probe_true_positives": 696,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.14993970659401568,
+      "full_absorption_rate": 0.1333907056798623,
+      "num_full_absorption": 155,
+      "num_probe_true_positives": 1162,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.12240690347298436,
+      "full_absorption_rate": 0.07788671023965142,
+      "num_full_absorption": 143,
+      "num_probe_true_positives": 1836,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.07982834734533917,
+      "full_absorption_rate": 0.06090133982947625,
+      "num_full_absorption": 50,
+      "num_probe_true_positives": 821,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.19346406307592456,
+      "full_absorption_rate": 0.21577946768060838,
+      "num_full_absorption": 227,
+      "num_probe_true_positives": 1052,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.5301150273469434,
+      "full_absorption_rate": 0.4553846153846154,
+      "num_full_absorption": 1036,
+      "num_probe_true_positives": 2275,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.01507865766197323,
+      "full_absorption_rate": 0.03076923076923077,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 195,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.2573794005707176,
+      "full_absorption_rate": 0.21096051856216852,
+      "num_full_absorption": 358,
+      "num_probe_true_positives": 1697,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.5040795030415306,
+      "full_absorption_rate": 0.4915720674234606,
+      "num_full_absorption": 1429,
+      "num_probe_true_positives": 2907,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.15276755271078457,
+      "full_absorption_rate": 0.08928571428571429,
+      "num_full_absorption": 150,
+      "num_probe_true_positives": 1680,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.22381275073634874,
+      "full_absorption_rate": 0.24521072796934865,
+      "num_full_absorption": 192,
+      "num_probe_true_positives": 783,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.026830089617880883,
+      "full_absorption_rate": 0.026717557251908396,
+      "num_full_absorption": 21,
+      "num_probe_true_positives": 786,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.07490041912777874,
+      "full_absorption_rate": 0.07849293563579278,
+      "num_full_absorption": 50,
+      "num_probe_true_positives": 637,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.05162827599896595,
+      "full_absorption_rate": 0.038834951456310676,
+      "num_full_absorption": 4,
+      "num_probe_true_positives": 103,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.042818109707770516,
+      "full_absorption_rate": 0.046242774566473986,
+      "num_full_absorption": 8,
+      "num_probe_true_positives": 173,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.001216099105256222,
+      "full_absorption_rate": 0.043478260869565216,
+      "num_full_absorption": 11,
+      "num_probe_true_positives": 253,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_4",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

random_seed_eval_results/autointerp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_0_custom_sae_eval_results.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cd264c6bcbb7a785af8168b12a582a0b00e8b6ec78f32f4df8a68432daf4c3e
+size 26730928

random_seed_eval_results/autointerp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_1_custom_sae_eval_results.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a22e244c1bd987a1dbf0b6048cc06a8d67a12488adc40a7c99cbc266996f2a41
+size 26750492

random_seed_eval_results/autointerp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_2_custom_sae_eval_results.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ffa3c81081876da4dad8b3ac18f00fe7a94d3fc56428680351a75bf3cff061f7
+size 26599749

random_seed_eval_results/autointerp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_3_custom_sae_eval_results.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8b84354daa6b2e2a7289ed82da6317b1ae7d598f5847285622afaa7a2d2549c
+size 26846759

random_seed_eval_results/autointerp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_4_custom_sae_eval_results.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82f546c16abf1b1eb58446c47e29a02b5e4143f01437db21e85faa030dd77276
+size 26752322

random_seed_eval_results/scr/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_0_custom_sae_eval_results.json

@@ -0,0 +1,323 @@
+{
+  "eval_type_id": "scr",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": true,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "9dee0d3c-103c-4d65-953a-0f4f306c5fe1",
+  "datetime_epoch_millis": 1738791866252,
+  "eval_result_metrics": {
+    "scr_metrics": {
+      "scr_dir1_threshold_2": 0.22513964187620733,
+      "scr_metric_threshold_2": 0.1385347875882101,
+      "scr_dir2_threshold_2": 0.14459702159963525,
+      "scr_dir1_threshold_5": 0.21983111343139253,
+      "scr_metric_threshold_5": 0.20326640785460648,
+      "scr_dir2_threshold_5": 0.21158442490747542,
+      "scr_dir1_threshold_10": 0.25192846118074297,
+      "scr_metric_threshold_10": 0.2749545350332416,
+      "scr_dir2_threshold_10": 0.2841028124566354,
+      "scr_dir1_threshold_20": 0.20284025239431253,
+      "scr_metric_threshold_20": 0.32735915703734847,
+      "scr_dir2_threshold_20": 0.33435385077642354,
+      "scr_dir1_threshold_50": 0.13344059547499754,
+      "scr_metric_threshold_50": 0.3972973820182854,
+      "scr_dir2_threshold_50": 0.39891958599459704,
+      "scr_dir1_threshold_100": 0.13881520081445878,
+      "scr_metric_threshold_100": 0.28708448453918034,
+      "scr_dir2_threshold_100": 0.3102170234845004,
+      "scr_dir1_threshold_500": -0.48690647479983323,
+      "scr_metric_threshold_500": 0.3190861754204293,
+      "scr_dir2_threshold_500": 0.3425712257082064
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_professor_nurse_results",
+      "scr_dir1_threshold_2": 0.43749988358469727,
+      "scr_metric_threshold_2": 0.022167494916722333,
+      "scr_dir2_threshold_2": 0.022167494916722333,
+      "scr_dir1_threshold_5": 0.4687504074535596,
+      "scr_metric_threshold_5": 0.039408798291137845,
+      "scr_dir2_threshold_5": 0.039408798291137845,
+      "scr_dir1_threshold_10": 0.4687504074535596,
+      "scr_metric_threshold_10": 0.07389155184943222,
+      "scr_dir2_threshold_10": 0.07389155184943222,
+      "scr_dir1_threshold_20": 0.42187508731147705,
+      "scr_metric_threshold_20": 0.07389155184943222,
+      "scr_dir2_threshold_20": 0.07389155184943222,
+      "scr_dir1_threshold_50": 0.43749988358469727,
+      "scr_metric_threshold_50": 0.15270929524117124,
+      "scr_dir2_threshold_50": 0.15270929524117124,
+      "scr_dir1_threshold_100": 0.3749997671693945,
+      "scr_metric_threshold_100": 0.21428558844903142,
+      "scr_dir2_threshold_100": 0.21428558844903142,
+      "scr_dir1_threshold_500": -1.406249359715835,
+      "scr_metric_threshold_500": 0.34236451321652195,
+      "scr_dir2_threshold_500": 0.34236451321652195
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
+      "scr_dir1_threshold_2": 0.2803735818828607,
+      "scr_metric_threshold_2": 0.2622478955438056,
+      "scr_dir2_threshold_2": 0.2622478955438056,
+      "scr_dir1_threshold_5": 0.2616822325784525,
+      "scr_metric_threshold_5": 0.33717591477669556,
+      "scr_dir2_threshold_5": 0.33717591477669556,
+      "scr_dir1_threshold_10": 0.2710279072306566,
+      "scr_metric_threshold_10": 0.41498556949231613,
+      "scr_dir2_threshold_10": 0.41498556949231613,
+      "scr_dir1_threshold_20": 0.12149544163702218,
+      "scr_metric_threshold_20": 0.4265129702797845,
+      "scr_dir2_threshold_20": 0.4265129702797845,
+      "scr_dir1_threshold_50": -0.46728986019089086,
+      "scr_metric_threshold_50": 0.4207493557717049,
+      "scr_dir2_threshold_50": 0.4207493557717049,
+      "scr_dir1_threshold_100": -0.7102807434649352,
+      "scr_metric_threshold_100": -0.02305480157493678,
+      "scr_dir2_threshold_100": -0.02305480157493678,
+      "scr_dir1_threshold_500": -1.4953274412002928,
+      "scr_metric_threshold_500": -0.11527366433206554,
+      "scr_dir2_threshold_500": -0.11527366433206554
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
+      "scr_dir1_threshold_2": 0.5312505238688624,
+      "scr_metric_threshold_2": 0.04556963247779803,
+      "scr_dir2_threshold_2": 0.04556963247779803,
+      "scr_dir1_threshold_5": 0.5156247962732202,
+      "scr_metric_threshold_5": 0.10379747943300446,
+      "scr_dir2_threshold_5": 0.10379747943300446,
+      "scr_dir1_threshold_10": 0.5468753201420825,
+      "scr_metric_threshold_10": 0.13670889743339407,
+      "scr_dir2_threshold_10": 0.13670889743339407,
+      "scr_dir1_threshold_20": 0.42187508731147705,
+      "scr_metric_threshold_20": 0.22278493695715454,
+      "scr_dir2_threshold_20": 0.22278493695715454,
+      "scr_dir1_threshold_50": 0.2656252619344312,
+      "scr_metric_threshold_50": 0.26582280582121537,
+      "scr_dir2_threshold_50": 0.26582280582121537,
+      "scr_dir1_threshold_100": 0.07812491268852294,
+      "scr_metric_threshold_100": -0.0025316127159178037,
+      "scr_dir2_threshold_100": -0.0025316127159178037,
+      "scr_dir1_threshold_500": -2.1406240977814037,
+      "scr_metric_threshold_500": -0.005063225431835607,
+      "scr_dir2_threshold_500": -0.005063225431835607
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
+      "scr_dir1_threshold_2": 0.259842390342736,
+      "scr_metric_threshold_2": 0.14836798243767776,
+      "scr_dir2_threshold_2": 0.14836798243767776,
+      "scr_dir1_threshold_5": -0.007873912274188802,
+      "scr_metric_threshold_5": 0.25816026467998676,
+      "scr_dir2_threshold_5": 0.25816026467998676,
+      "scr_dir1_threshold_10": -0.04724394297291932,
+      "scr_metric_threshold_10": 0.32047488451379413,
+      "scr_dir2_threshold_10": 0.32047488451379413,
+      "scr_dir1_threshold_20": -0.2283462719181943,
+      "scr_metric_threshold_20": 0.28486650505613625,
+      "scr_dir2_threshold_20": 0.28486650505613625,
+      "scr_dir1_threshold_50": -0.13385791664456914,
+      "scr_metric_threshold_50": 0.3916914665607343,
+      "scr_dir2_threshold_50": 0.3916914665607343,
+      "scr_dir1_threshold_100": 0.40157468858928047,
+      "scr_metric_threshold_100": 0.09792299929146529,
+      "scr_dir2_threshold_100": 0.09792299929146529,
+      "scr_dir1_threshold_500": 0.14960621052073325,
+      "scr_metric_threshold_500": 0.18100904115762476,
+      "scr_dir2_threshold_500": 0.18100904115762476
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
+      "scr_dir1_threshold_2": 0.021739172687599125,
+      "scr_metric_threshold_2": 0.13333325541876354,
+      "scr_dir2_threshold_2": 0.13333325541876354,
+      "scr_dir1_threshold_5": 0.048913057562533044,
+      "scr_metric_threshold_5": 0.22745086122869718,
+      "scr_dir2_threshold_5": 0.22745086122869718,
+      "scr_dir1_threshold_10": 0.11413025168707046,
+      "scr_metric_threshold_10": 0.4470586860331121,
+      "scr_dir2_threshold_10": 0.4470586860331121,
+      "scr_dir1_threshold_20": 0.05434776974986783,
+      "scr_metric_threshold_20": 0.5960784817878589,
+      "scr_dir2_threshold_20": 0.5960784817878589,
+      "scr_dir1_threshold_50": -0.048913057562533044,
+      "scr_metric_threshold_50": 0.6862745693856515,
+      "scr_dir2_threshold_50": 0.6862745693856515,
+      "scr_dir1_threshold_100": -0.021739172687599125,
+      "scr_metric_threshold_100": 0.7568625984353196,
+      "scr_dir2_threshold_100": 0.7568625984353196,
+      "scr_dir1_threshold_500": 0.010869424374669583,
+      "scr_metric_threshold_500": 0.7411765255867552,
+      "scr_dir2_threshold_500": 0.7411765255867552
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
+      "scr_dir1_threshold_2": 0.09278322958848217,
+      "scr_metric_threshold_2": 0.08064531634921589,
+      "scr_dir2_threshold_2": 0.08064531634921589,
+      "scr_dir1_threshold_5": 0.2010307916356866,
+      "scr_metric_threshold_5": 0.12096785435318151,
+      "scr_dir2_threshold_5": 0.12096785435318151,
+      "scr_dir1_threshold_10": 0.25257713270943044,
+      "scr_metric_threshold_10": 0.16129039235714715,
+      "scr_dir2_threshold_10": 0.16129039235714715,
+      "scr_dir1_threshold_20": 0.3350515242197629,
+      "scr_metric_threshold_20": 0.23387105690079912,
+      "scr_dir2_threshold_20": 0.23387105690079912,
+      "scr_dir1_threshold_50": 0.3195874990014686,
+      "scr_metric_threshold_50": 0.3588711169861203,
+      "scr_dir2_threshold_50": 0.3588711169861203,
+      "scr_dir1_threshold_100": 0.34536066953834055,
+      "scr_metric_threshold_100": 0.47983873099801716,
+      "scr_dir2_threshold_100": 0.47983873099801716,
+      "scr_dir1_threshold_500": 0.2731957305870136,
+      "scr_metric_threshold_500": 0.5604838070059485,
+      "scr_dir2_threshold_500": 0.5604838070059485
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
+      "scr_dir1_threshold_2": 0.12612611161318626,
+      "scr_metric_threshold_2": 0.3644444821204628,
+      "scr_dir2_threshold_2": 0.3644444821204628,
+      "scr_dir1_threshold_5": 0.18468472096703434,
+      "scr_metric_threshold_5": 0.45333327681930574,
+      "scr_dir2_threshold_5": 0.45333327681930574,
+      "scr_dir1_threshold_10": 0.3063063861274755,
+      "scr_metric_threshold_10": 0.5422220715181487,
+      "scr_dir2_threshold_10": 0.5422220715181487,
+      "scr_dir1_threshold_20": 0.3333333333333333,
+      "scr_metric_threshold_20": 0.6177777059578677,
+      "scr_dir2_threshold_20": 0.6177777059578677,
+      "scr_dir1_threshold_50": 0.445945837098897,
+      "scr_metric_threshold_50": 0.6533333298012066,
+      "scr_dir2_threshold_50": 0.6533333298012066,
+      "scr_dir1_threshold_100": 0.5180180543003676,
+      "scr_metric_threshold_100": 0.6488889430481654,
+      "scr_dir2_threshold_100": 0.6488889430481654,
+      "scr_dir1_threshold_500": 0.5630630558065931,
+      "scr_metric_threshold_500": 0.697777727150628,
+      "scr_dir2_threshold_500": 0.697777727150628
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
+      "scr_dir1_threshold_2": 0.05150224144123495,
+      "scr_metric_threshold_2": 0.05150224144123495,
+      "scr_dir2_threshold_2": 0.10000011353263609,
+      "scr_dir1_threshold_5": 0.08583681325484281,
+      "scr_metric_threshold_5": 0.08583681325484281,
+      "scr_dir2_threshold_5": 0.1523809496777944,
+      "scr_dir1_threshold_10": 0.10300422706858779,
+      "scr_metric_threshold_10": 0.10300422706858779,
+      "scr_dir2_threshold_10": 0.17619044645573817,
+      "scr_dir1_threshold_20": 0.16309004750975417,
+      "scr_metric_threshold_20": 0.16309004750975417,
+      "scr_dir2_threshold_20": 0.21904759742235502,
+      "scr_dir1_threshold_50": 0.2489271165784791,
+      "scr_metric_threshold_50": 0.2489271165784791,
+      "scr_dir2_threshold_50": 0.2619047483889719,
+      "scr_dir1_threshold_100": 0.1244634303822985,
+      "scr_metric_threshold_100": 0.1244634303822985,
+      "scr_dir2_threshold_100": 0.3095237419448595,
+      "scr_dir1_threshold_500": 0.15021467900985702,
+      "scr_metric_threshold_500": 0.15021467900985702,
+      "scr_dir2_threshold_500": 0.33809508131207394
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_0",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

random_seed_eval_results/scr/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_1_custom_sae_eval_results.json

@@ -0,0 +1,323 @@
+{
+  "eval_type_id": "scr",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": true,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "60baa903-f878-4b23-a5cd-ec96f2332e41",
+  "datetime_epoch_millis": 1738791541544,
+  "eval_result_metrics": {
+    "scr_metrics": {
+      "scr_dir1_threshold_2": 0.2361877325388411,
+      "scr_metric_threshold_2": 0.13791798180534542,
+      "scr_dir2_threshold_2": 0.1371311321790473,
+      "scr_dir1_threshold_5": 0.22353532292083783,
+      "scr_metric_threshold_5": 0.19684522910292013,
+      "scr_dir2_threshold_5": 0.20170931003047823,
+      "scr_dir1_threshold_10": 0.2429635217288672,
+      "scr_metric_threshold_10": 0.27304872008261916,
+      "scr_dir2_threshold_10": 0.276714683405259,
+      "scr_dir1_threshold_20": 0.2338835500774749,
+      "scr_metric_threshold_20": 0.3322860879225477,
+      "scr_dir2_threshold_20": 0.33648086797508275,
+      "scr_dir1_threshold_50": 0.30544138126010734,
+      "scr_metric_threshold_50": 0.3970974400299999,
+      "scr_dir2_threshold_50": 0.39961631797060004,
+      "scr_dir1_threshold_100": 0.16496334174074267,
+      "scr_metric_threshold_100": 0.290651594443317,
+      "scr_dir2_threshold_100": 0.30121004864569334,
+      "scr_dir1_threshold_500": -0.3407507620134751,
+      "scr_metric_threshold_500": 0.3261204064251065,
+      "scr_dir2_threshold_500": 0.34041889450148693
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_professor_nurse_results",
+      "scr_dir1_threshold_2": 0.43749988358469727,
+      "scr_metric_threshold_2": 0.024630517283144072,
+      "scr_dir2_threshold_2": 0.024630517283144072,
+      "scr_dir1_threshold_5": 0.4843752037267798,
+      "scr_metric_threshold_5": 0.049261034566288144,
+      "scr_dir2_threshold_5": 0.049261034566288144,
+      "scr_dir1_threshold_10": 0.5156247962732202,
+      "scr_metric_threshold_10": 0.06650233794070366,
+      "scr_dir2_threshold_10": 0.06650233794070366,
+      "scr_dir1_threshold_20": 0.4062502910382569,
+      "scr_metric_threshold_20": 0.10591128304130484,
+      "scr_dir2_threshold_20": 0.10591128304130484,
+      "scr_dir1_threshold_50": 0.4062502910382569,
+      "scr_metric_threshold_50": 0.13793101423317747,
+      "scr_dir2_threshold_50": 0.13793101423317747,
+      "scr_dir1_threshold_100": 0.4062502910382569,
+      "scr_metric_threshold_100": 0.16748757624916502,
+      "scr_dir2_threshold_100": 0.16748757624916502,
+      "scr_dir1_threshold_500": -1.406249359715835,
+      "scr_metric_threshold_500": 0.23891625254163884,
+      "scr_dir2_threshold_500": 0.23891625254163884
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
+      "scr_dir1_threshold_2": 0.2803735818828607,
+      "scr_metric_threshold_2": 0.273775296331274,
+      "scr_dir2_threshold_2": 0.273775296331274,
+      "scr_dir1_threshold_5": 0.2616822325784525,
+      "scr_metric_threshold_5": 0.291066311626822,
+      "scr_dir2_threshold_5": 0.291066311626822,
+      "scr_dir1_threshold_10": 0.2616822325784525,
+      "scr_metric_threshold_10": 0.36311235183436297,
+      "scr_dir2_threshold_10": 0.36311235183436297,
+      "scr_dir1_threshold_20": 0.2336446515690506,
+      "scr_metric_threshold_20": 0.4582133653881499,
+      "scr_dir2_threshold_20": 0.4582133653881499,
+      "scr_dir1_threshold_50": 0.35514009320607276,
+      "scr_metric_threshold_50": 0.4351585638132131,
+      "scr_dir2_threshold_50": 0.4351585638132131,
+      "scr_dir1_threshold_100": -0.514018790504701,
+      "scr_metric_threshold_100": -0.01152740078746839,
+      "scr_dir2_threshold_100": -0.01152740078746839,
+      "scr_dir1_threshold_500": -1.1682243719508323,
+      "scr_metric_threshold_500": -0.1815560900315269,
+      "scr_dir2_threshold_500": -0.1815560900315269
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
+      "scr_dir1_threshold_2": 0.5625001164153027,
+      "scr_metric_threshold_2": 0.037974794330044616,
+      "scr_dir2_threshold_2": 0.037974794330044616,
+      "scr_dir1_threshold_5": 0.5625001164153027,
+      "scr_metric_threshold_5": 0.08354442680784264,
+      "scr_dir2_threshold_5": 0.08354442680784264,
+      "scr_dir1_threshold_10": 0.5156247962732202,
+      "scr_metric_threshold_10": 0.15696210095637528,
+      "scr_dir2_threshold_10": 0.15696210095637528,
+      "scr_dir1_threshold_20": 0.43749988358469727,
+      "scr_metric_threshold_20": 0.20000012071825551,
+      "scr_dir2_threshold_20": 0.20000012071825551,
+      "scr_dir1_threshold_50": 0.2968748544808716,
+      "scr_metric_threshold_50": 0.28101278391236095,
+      "scr_dir2_threshold_50": 0.28101278391236095,
+      "scr_dir1_threshold_100": 0.2343756693879908,
+      "scr_metric_threshold_100": 0.04303801976188022,
+      "scr_dir2_threshold_100": 0.04303801976188022,
+      "scr_dir1_threshold_500": -1.421874155989055,
+      "scr_metric_threshold_500": 0.015189978091145603,
+      "scr_dir2_threshold_500": 0.015189978091145603
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
+      "scr_dir1_threshold_2": 0.2440945657943584,
+      "scr_metric_threshold_2": 0.18100904115762476,
+      "scr_dir2_threshold_2": 0.18100904115762476,
+      "scr_dir1_threshold_5": -0.03149611842454171,
+      "scr_metric_threshold_5": 0.2759645428430035,
+      "scr_dir2_threshold_5": 0.2759645428430035,
+      "scr_dir1_threshold_10": -0.10236179822002743,
+      "scr_metric_threshold_10": 0.3382789858084353,
+      "scr_dir2_threshold_10": 0.3382789858084353,
+      "scr_dir1_threshold_20": -0.16535403506911087,
+      "scr_metric_threshold_20": 0.32640952598921597,
+      "scr_dir2_threshold_20": 0.32640952598921597,
+      "scr_dir1_threshold_50": 0.37007903949252524,
+      "scr_metric_threshold_50": 0.4272998460183922,
+      "scr_dir2_threshold_50": 0.4272998460183922,
+      "scr_dir1_threshold_100": 0.22834674124598078,
+      "scr_metric_threshold_100": 0.1364985226184585,
+      "scr_dir2_threshold_100": 0.1364985226184585,
+      "scr_dir1_threshold_500": 0.10236226754781394,
+      "scr_metric_threshold_500": 0.19287850097684403,
+      "scr_dir2_threshold_500": 0.19287850097684403
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
+      "scr_dir1_threshold_2": 0.08152165462480175,
+      "scr_metric_threshold_2": 0.13333325541876354,
+      "scr_dir2_threshold_2": 0.13333325541876354,
+      "scr_dir1_threshold_5": 0.06521719412453741,
+      "scr_metric_threshold_5": 0.262744992625386,
+      "scr_dir2_threshold_5": 0.262744992625386,
+      "scr_dir1_threshold_10": 0.10326082731240087,
+      "scr_metric_threshold_10": 0.4784312992176598,
+      "scr_dir2_threshold_10": 0.4784312992176598,
+      "scr_dir1_threshold_20": 0.07608694243746696,
+      "scr_metric_threshold_20": 0.6039215182121411,
+      "scr_dir2_threshold_20": 0.6039215182121411,
+      "scr_dir1_threshold_50": -0.04347834537519825,
+      "scr_metric_threshold_50": 0.7254902189944813,
+      "scr_dir2_threshold_50": 0.7254902189944813,
+      "scr_dir1_threshold_100": -0.032608921000528666,
+      "scr_metric_threshold_100": 0.780392175195585,
+      "scr_dir2_threshold_100": 0.780392175195585,
+      "scr_dir1_threshold_500": 0.10326082731240087,
+      "scr_metric_threshold_500": 0.7411765255867552,
+      "scr_dir2_threshold_500": 0.7411765255867552
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
+      "scr_dir1_threshold_2": 0.0979381094881989,
+      "scr_metric_threshold_2": 0.06854845881151235,
+      "scr_dir2_threshold_2": 0.06854845881151235,
+      "scr_dir1_threshold_5": 0.18041219375810347,
+      "scr_metric_threshold_5": 0.10887099681547797,
+      "scr_dir2_threshold_5": 0.10887099681547797,
+      "scr_dir1_threshold_10": 0.24226798739085284,
+      "scr_metric_threshold_10": 0.1370969176230247,
+      "scr_dir2_threshold_10": 0.1370969176230247,
+      "scr_dir1_threshold_20": 0.3195874990014686,
+      "scr_metric_threshold_20": 0.2056451360932524,
+      "scr_dir2_threshold_20": 0.2056451360932524,
+      "scr_dir1_threshold_50": 0.34020609687905173,
+      "scr_metric_threshold_50": 0.30241927537102686,
+      "scr_dir2_threshold_50": 0.30241927537102686,
+      "scr_dir1_threshold_100": 0.381443292634218,
+      "scr_metric_threshold_100": 0.3588711169861203,
+      "scr_dir2_threshold_100": 0.3588711169861203,
+      "scr_dir1_threshold_500": 0.3659792674159237,
+      "scr_metric_threshold_500": 0.6250000600853212,
+      "scr_dir2_threshold_500": 0.6250000600853212
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
+      "scr_dir1_threshold_2": 0.11261250376556366,
+      "scr_metric_threshold_2": 0.3111110463554545,
+      "scr_dir2_threshold_2": 0.3111110463554545,
+      "scr_dir1_threshold_5": 0.17117111311941172,
+      "scr_metric_threshold_5": 0.4088888794698843,
+      "scr_dir2_threshold_5": 0.4088888794698843,
+      "scr_dir1_threshold_10": 0.2702702775267402,
+      "scr_metric_threshold_10": 0.5066667125843141,
+      "scr_dir2_threshold_10": 0.5066667125843141,
+      "scr_dir1_threshold_20": 0.38738749623443636,
+      "scr_metric_threshold_20": 0.5822220821145289,
+      "scr_dir2_threshold_20": 0.5822220821145289,
+      "scr_dir1_threshold_50": 0.4909911070945099,
+      "scr_metric_threshold_50": 0.6399999046325785,
+      "scr_dir2_threshold_50": 0.6399999046325785,
+      "scr_dir1_threshold_100": 0.4099099969875491,
+      "scr_metric_threshold_100": 0.6444442913856198,
+      "scr_dir2_threshold_100": 0.6444442913856198,
+      "scr_dir1_threshold_500": 0.43693694419340684,
+      "scr_metric_threshold_500": 0.7155555390722975,
+      "scr_dir2_threshold_500": 0.7155555390722975
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
+      "scr_dir1_threshold_2": 0.07296144475494565,
+      "scr_metric_threshold_2": 0.07296144475494565,
+      "scr_dir2_threshold_2": 0.06666664774456064,
+      "scr_dir1_threshold_5": 0.09442064806865635,
+      "scr_metric_threshold_5": 0.09442064806865635,
+      "scr_dir2_threshold_5": 0.1333332954891213,
+      "scr_dir1_threshold_10": 0.13733905469607777,
+      "scr_metric_threshold_10": 0.13733905469607777,
+      "scr_dir2_threshold_10": 0.16666676127719673,
+      "scr_dir1_threshold_20": 0.17596567182353345,
+      "scr_metric_threshold_20": 0.17596567182353345,
+      "scr_dir2_threshold_20": 0.20952391224381361,
+      "scr_dir1_threshold_50": 0.2274679132647684,
+      "scr_metric_threshold_50": 0.2274679132647684,
+      "scr_dir2_threshold_50": 0.24761893678956953,
+      "scr_dir1_threshold_100": 0.20600845413717558,
+      "scr_metric_threshold_100": 0.20600845413717558,
+      "scr_dir2_threshold_100": 0.2904760877561864,
+      "scr_dir1_threshold_500": 0.2618024850783763,
+      "scr_metric_threshold_500": 0.2618024850783763,
+      "scr_dir2_threshold_500": 0.37619038968942015
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_1",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

random_seed_eval_results/scr/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_2_custom_sae_eval_results.json

@@ -0,0 +1,323 @@
+{
+  "eval_type_id": "scr",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": true,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "abc3a7fe-901e-41b7-b863-d7c2fe187f5d",
+  "datetime_epoch_millis": 1738792191381,
+  "eval_result_metrics": {
+    "scr_metrics": {
+      "scr_dir1_threshold_2": 0.27189662143546267,
+      "scr_metric_threshold_2": 0.14004249795768345,
+      "scr_dir2_threshold_2": 0.1368670346002006,
+      "scr_dir1_threshold_5": 0.31141524878678495,
+      "scr_metric_threshold_5": 0.2115742250577737,
+      "scr_dir2_threshold_5": 0.211903765527429,
+      "scr_dir1_threshold_10": 0.3010599506808038,
+      "scr_metric_threshold_10": 0.29065234137699436,
+      "scr_dir2_threshold_10": 0.2925836779282715,
+      "scr_dir1_threshold_20": 0.25286159416685644,
+      "scr_metric_threshold_20": 0.3374504224840684,
+      "scr_dir2_threshold_20": 0.33961678908221155,
+      "scr_dir1_threshold_50": 0.32881244000862797,
+      "scr_metric_threshold_50": 0.385774710992042,
+      "scr_dir2_threshold_50": 0.3761384965816031,
+      "scr_dir1_threshold_100": 0.1704072584532131,
+      "scr_metric_threshold_100": 0.3010885087468888,
+      "scr_dir2_threshold_100": 0.29610434806667896,
+      "scr_dir1_threshold_500": -0.3535588273019515,
+      "scr_metric_threshold_500": 0.32484492101420326,
+      "scr_dir2_threshold_500": 0.33724782632593275
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_professor_nurse_results",
+      "scr_dir1_threshold_2": 0.42187508731147705,
+      "scr_metric_threshold_2": 0.02709353964956581,
+      "scr_dir2_threshold_2": 0.02709353964956581,
+      "scr_dir1_threshold_5": 0.42187508731147705,
+      "scr_metric_threshold_5": 0.044334989833444666,
+      "scr_dir2_threshold_5": 0.044334989833444666,
+      "scr_dir1_threshold_10": 0.4062502910382569,
+      "scr_metric_threshold_10": 0.08128076575816078,
+      "scr_dir2_threshold_10": 0.08128076575816078,
+      "scr_dir1_threshold_20": 0.4062502910382569,
+      "scr_metric_threshold_20": 0.12315258641572036,
+      "scr_dir2_threshold_20": 0.12315258641572036,
+      "scr_dir1_threshold_50": 0.4062502910382569,
+      "scr_metric_threshold_50": 0.2635467698247829,
+      "scr_dir2_threshold_50": 0.2635467698247829,
+      "scr_dir1_threshold_100": 0.42187508731147705,
+      "scr_metric_threshold_100": 0.3349752993077934,
+      "scr_dir2_threshold_100": 0.3349752993077934,
+      "scr_dir1_threshold_500": -1.01562479627322,
+      "scr_metric_threshold_500": 0.40147778405796036,
+      "scr_dir2_threshold_500": 0.40147778405796036
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
+      "scr_dir1_threshold_2": 0.2803735818828607,
+      "scr_metric_threshold_2": 0.273775296331274,
+      "scr_dir2_threshold_2": 0.273775296331274,
+      "scr_dir1_threshold_5": 0.35514009320607276,
+      "scr_metric_threshold_5": 0.35158512281820375,
+      "scr_dir2_threshold_5": 0.35158512281820375,
+      "scr_dir1_threshold_10": 0.36448576785827685,
+      "scr_metric_threshold_10": 0.41498556949231613,
+      "scr_dir2_threshold_10": 0.41498556949231613,
+      "scr_dir1_threshold_20": 0.2990654882400586,
+      "scr_metric_threshold_20": 0.4899135887252061,
+      "scr_dir2_threshold_20": 0.4899135887252061,
+      "scr_dir1_threshold_50": 0.35514009320607276,
+      "scr_metric_threshold_50": 0.38328534615525994,
+      "scr_dir2_threshold_50": 0.38328534615525994,
+      "scr_dir1_threshold_100": -0.6635518131511251,
+      "scr_metric_threshold_100": -0.09221903452843794,
+      "scr_dir2_threshold_100": -0.09221903452843794,
+      "scr_dir1_threshold_500": -1.4299066045292848,
+      "scr_metric_threshold_500": -0.24495670847694848,
+      "scr_dir2_threshold_500": -0.24495670847694848
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
+      "scr_dir1_threshold_2": 0.5468753201420825,
+      "scr_metric_threshold_2": 0.037974794330044616,
+      "scr_dir2_threshold_2": 0.037974794330044616,
+      "scr_dir1_threshold_5": 0.5625001164153027,
+      "scr_metric_threshold_5": 0.08860765223967824,
+      "scr_dir2_threshold_5": 0.08860765223967824,
+      "scr_dir1_threshold_10": 0.5468753201420825,
+      "scr_metric_threshold_10": 0.13924051014931188,
+      "scr_dir2_threshold_10": 0.13924051014931188,
+      "scr_dir1_threshold_20": 0.28125005820765137,
+      "scr_metric_threshold_20": 0.20253173343417333,
+      "scr_dir2_threshold_20": 0.20253173343417333,
+      "scr_dir1_threshold_50": 0.39062549476503666,
+      "scr_metric_threshold_50": 0.25063297862788914,
+      "scr_dir2_threshold_50": 0.25063297862788914,
+      "scr_dir1_threshold_100": 0.2343756693879908,
+      "scr_metric_threshold_100": 0.05316462152337083,
+      "scr_dir2_threshold_100": 0.05316462152337083,
+      "scr_dir1_threshold_500": -1.2656243306120092,
+      "scr_metric_threshold_500": 0.037974794330044616,
+      "scr_dir2_threshold_500": 0.037974794330044616
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
+      "scr_dir1_threshold_2": 0.2519684780685472,
+      "scr_metric_threshold_2": 0.15430280078147524,
+      "scr_dir2_threshold_2": 0.15430280078147524,
+      "scr_dir1_threshold_5": 0.33070853946600826,
+      "scr_metric_threshold_5": 0.2640950830237842,
+      "scr_dir2_threshold_5": 0.2640950830237842,
+      "scr_dir1_threshold_10": 0.08661444299943634,
+      "scr_metric_threshold_10": 0.34421380415223274,
+      "scr_dir2_threshold_10": 0.34421380415223274,
+      "scr_dir1_threshold_20": -0.11023617982200275,
+      "scr_metric_threshold_20": 0.2908013233999337,
+      "scr_dir2_threshold_20": 0.2908013233999337,
+      "scr_dir1_threshold_50": 0.40944907019125576,
+      "scr_metric_threshold_50": 0.40059360564224267,
+      "scr_dir2_threshold_50": 0.40059360564224267,
+      "scr_dir1_threshold_100": 0.33858292106798354,
+      "scr_metric_threshold_100": 0.1988131424522659,
+      "scr_dir2_threshold_100": 0.1988131424522659,
+      "scr_dir1_threshold_500": 0.11811056142397805,
+      "scr_metric_threshold_500": 0.2997032856130665,
+      "scr_dir2_threshold_500": 0.2997032856130665
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
+      "scr_dir1_threshold_2": 0.048913057562533044,
+      "scr_metric_threshold_2": 0.1490195620110374,
+      "scr_dir2_threshold_2": 0.1490195620110374,
+      "scr_dir1_threshold_5": 0.06521719412453741,
+      "scr_metric_threshold_5": 0.28627433564194205,
+      "scr_dir2_threshold_5": 0.28627433564194205,
+      "scr_dir1_threshold_10": 0.08695636681213655,
+      "scr_metric_threshold_10": 0.49019608759779254,
+      "scr_dir2_threshold_10": 0.49019608759779254,
+      "scr_dir1_threshold_20": 0.05434776974986783,
+      "scr_metric_threshold_20": 0.6,
+      "scr_dir2_threshold_20": 0.6,
+      "scr_dir1_threshold_50": -0.07065223025013216,
+      "scr_metric_threshold_50": 0.7176469488264897,
+      "scr_dir2_threshold_50": 0.7176469488264897,
+      "scr_dir1_threshold_100": -0.06521751806279738,
+      "scr_metric_threshold_100": 0.7529410802231785,
+      "scr_dir2_threshold_100": 0.7529410802231785,
+      "scr_dir1_threshold_500": -0.04347834537519825,
+      "scr_metric_threshold_500": 0.7176469488264897,
+      "scr_dir2_threshold_500": 0.7176469488264897
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
+      "scr_dir1_threshold_2": 0.21649481685398092,
+      "scr_metric_threshold_2": 0.06451625307937271,
+      "scr_dir2_threshold_2": 0.06451625307937271,
+      "scr_dir1_threshold_5": 0.26804115792772476,
+      "scr_metric_threshold_5": 0.12096785435318151,
+      "scr_dir2_threshold_5": 0.12096785435318151,
+      "scr_dir1_threshold_10": 0.34536066953834055,
+      "scr_metric_threshold_10": 0.17741945562699032,
+      "scr_dir2_threshold_10": 0.17741945562699032,
+      "scr_dir1_threshold_20": 0.4175256084896675,
+      "scr_metric_threshold_20": 0.2137097878988163,
+      "scr_dir2_threshold_20": 0.2137097878988163,
+      "scr_dir1_threshold_50": 0.43814420636725065,
+      "scr_metric_threshold_50": 0.2943548639067476,
+      "scr_dir2_threshold_50": 0.2943548639067476,
+      "scr_dir1_threshold_100": 0.43814420636725065,
+      "scr_metric_threshold_100": 0.38306459172024276,
+      "scr_dir2_threshold_100": 0.38306459172024276,
+      "scr_dir1_threshold_500": 0.453608231585545,
+      "scr_metric_threshold_500": 0.5766128702757917,
+      "scr_dir2_threshold_500": 0.5766128702757917
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
+      "scr_dir1_threshold_2": 0.29279277827985295,
+      "scr_metric_threshold_2": 0.29777788609633066,
+      "scr_dir2_threshold_2": 0.29777788609633066,
+      "scr_dir1_threshold_5": 0.3333333333333333,
+      "scr_metric_threshold_5": 0.38222229404213226,
+      "scr_dir2_threshold_5": 0.38222229404213226,
+      "scr_dir1_threshold_10": 0.38738749623443636,
+      "scr_metric_threshold_10": 0.49333328741568594,
+      "scr_dir2_threshold_10": 0.49333328741568594,
+      "scr_dir1_threshold_20": 0.4729730527941422,
+      "scr_metric_threshold_20": 0.5777776953614876,
+      "scr_dir2_threshold_20": 0.5777776953614876,
+      "scr_dir1_threshold_50": 0.4054052820454166,
+      "scr_metric_threshold_50": 0.47999986224705776,
+      "scr_dir2_threshold_50": 0.47999986224705776,
+      "scr_dir1_threshold_100": 0.41441444344029416,
+      "scr_metric_threshold_100": 0.5333332980120661,
+      "scr_dir2_threshold_100": 0.5333332980120661,
+      "scr_dir1_threshold_500": 0.14414416591355395,
+      "scr_metric_threshold_500": 0.5999998940361982,
+      "scr_dir2_threshold_500": 0.5999998940361982
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
+      "scr_dir1_threshold_2": 0.11587985138236706,
+      "scr_metric_threshold_2": 0.11587985138236706,
+      "scr_dir2_threshold_2": 0.09047614452250444,
+      "scr_dir1_threshold_5": 0.15450646850982275,
+      "scr_metric_threshold_5": 0.15450646850982275,
+      "scr_dir2_threshold_5": 0.1571427922670651,
+      "scr_dir1_threshold_10": 0.18454925082346488,
+      "scr_metric_threshold_10": 0.18454925082346488,
+      "scr_dir2_threshold_10": 0.19999994323368195,
+      "scr_dir1_threshold_20": 0.20171666463720986,
+      "scr_metric_threshold_20": 0.20171666463720986,
+      "scr_dir2_threshold_20": 0.21904759742235502,
+      "scr_dir1_threshold_50": 0.29613731270586624,
+      "scr_metric_threshold_50": 0.29613731270586624,
+      "scr_dir2_threshold_50": 0.21904759742235502,
+      "scr_dir1_threshold_100": 0.24463507126463127,
+      "scr_metric_threshold_100": 0.24463507126463127,
+      "scr_dir2_threshold_100": 0.20476178582295265,
+      "scr_dir1_threshold_500": 0.21030049945102341,
+      "scr_metric_threshold_500": 0.21030049945102341,
+      "scr_dir2_threshold_500": 0.3095237419448595
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_2",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

random_seed_eval_results/scr/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_3_custom_sae_eval_results.json

@@ -0,0 +1,323 @@
+{
+  "eval_type_id": "scr",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": true,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "ba20839f-14dc-439c-bb3f-cfa46001940a",
+  "datetime_epoch_millis": 1738792516361,
+  "eval_result_metrics": {
+    "scr_metrics": {
+      "scr_dir1_threshold_2": 0.2122048197936822,
+      "scr_metric_threshold_2": 0.1276442565195558,
+      "scr_dir2_threshold_2": 0.13525715145509146,
+      "scr_dir1_threshold_5": 0.20309333729784668,
+      "scr_metric_threshold_5": 0.1880617094849222,
+      "scr_dir2_threshold_5": 0.19781287418957544,
+      "scr_dir1_threshold_10": 0.2087873066702555,
+      "scr_metric_threshold_10": 0.25796852124284025,
+      "scr_dir2_threshold_10": 0.28046238714353955,
+      "scr_dir1_threshold_20": 0.2119416002085242,
+      "scr_metric_threshold_20": 0.30689936831597747,
+      "scr_dir2_threshold_20": 0.32259525025751606,
+      "scr_dir1_threshold_50": 0.301772100798797,
+      "scr_metric_threshold_50": 0.3936194698709037,
+      "scr_dir2_threshold_50": 0.40370783200281274,
+      "scr_dir1_threshold_100": 0.06892208153648197,
+      "scr_metric_threshold_100": 0.30548528244548945,
+      "scr_dir2_threshold_100": 0.32957328747021003,
+      "scr_dir1_threshold_500": -0.5674361096258931,
+      "scr_metric_threshold_500": 0.31796247845360437,
+      "scr_dir2_threshold_500": 0.3613484461501164
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_professor_nurse_results",
+      "scr_dir1_threshold_2": 0.42187508731147705,
+      "scr_metric_threshold_2": 0.024630517283144072,
+      "scr_dir2_threshold_2": 0.024630517283144072,
+      "scr_dir1_threshold_5": 0.4843752037267798,
+      "scr_metric_threshold_5": 0.039408798291137845,
+      "scr_dir2_threshold_5": 0.039408798291137845,
+      "scr_dir1_threshold_10": 0.5156247962732202,
+      "scr_metric_threshold_10": 0.0566502484750167,
+      "scr_dir2_threshold_10": 0.0566502484750167,
+      "scr_dir1_threshold_20": 0.4843752037267798,
+      "scr_metric_threshold_20": 0.07389155184943222,
+      "scr_dir2_threshold_20": 0.07389155184943222,
+      "scr_dir1_threshold_50": 0.42187508731147705,
+      "scr_metric_threshold_50": 0.16256153151632155,
+      "scr_dir2_threshold_50": 0.16256153151632155,
+      "scr_dir1_threshold_100": 0.39062549476503666,
+      "scr_metric_threshold_100": 0.3029555681159207,
+      "scr_dir2_threshold_100": 0.3029555681159207,
+      "scr_dir1_threshold_500": -1.2656243306120092,
+      "scr_metric_threshold_500": 0.3817733115076598,
+      "scr_dir2_threshold_500": 0.3817733115076598
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
+      "scr_dir1_threshold_2": 0.2803735818828607,
+      "scr_metric_threshold_2": 0.20461106337777285,
+      "scr_dir2_threshold_2": 0.20461106337777285,
+      "scr_dir1_threshold_5": 0.2897198135878545,
+      "scr_metric_threshold_5": 0.21902027141928104,
+      "scr_dir2_threshold_5": 0.21902027141928104,
+      "scr_dir1_threshold_10": 0.2616822325784525,
+      "scr_metric_threshold_10": 0.273775296331274,
+      "scr_dir2_threshold_10": 0.273775296331274,
+      "scr_dir1_threshold_20": 0.2242989769168465,
+      "scr_metric_threshold_20": 0.35158512281820375,
+      "scr_dir2_threshold_20": 0.35158512281820375,
+      "scr_dir1_threshold_50": 0.2242989769168465,
+      "scr_metric_threshold_50": 0.37463975262183136,
+      "scr_dir2_threshold_50": 0.37463975262183136,
+      "scr_dir1_threshold_100": -0.9345797203817817,
+      "scr_metric_threshold_100": -0.014409208041508192,
+      "scr_dir2_threshold_100": -0.014409208041508192,
+      "scr_dir1_threshold_500": -1.1962619529602343,
+      "scr_metric_threshold_500": -0.1325648513989227,
+      "scr_dir2_threshold_500": -0.1325648513989227
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
+      "scr_dir1_threshold_2": 0.5468753201420825,
+      "scr_metric_threshold_2": 0.04810139609153522,
+      "scr_dir2_threshold_2": 0.04810139609153522,
+      "scr_dir1_threshold_5": 0.5468753201420825,
+      "scr_metric_threshold_5": 0.09367087767151386,
+      "scr_dir2_threshold_5": 0.09367087767151386,
+      "scr_dir1_threshold_10": 0.5468753201420825,
+      "scr_metric_threshold_10": 0.1721519281497015,
+      "scr_dir2_threshold_10": 0.1721519281497015,
+      "scr_dir1_threshold_20": 0.5156247962732202,
+      "scr_metric_threshold_20": 0.20759495886600893,
+      "scr_dir2_threshold_20": 0.20759495886600893,
+      "scr_dir1_threshold_50": 0.4687504074535596,
+      "scr_metric_threshold_50": 0.29113923477603215,
+      "scr_dir2_threshold_50": 0.29113923477603215,
+      "scr_dir1_threshold_100": 0.21874994179234863,
+      "scr_metric_threshold_100": 0.022784816238899015,
+      "scr_dir2_threshold_100": 0.022784816238899015,
+      "scr_dir1_threshold_500": -2.8906245634426146,
+      "scr_metric_threshold_500": 0.0075949890455728015,
+      "scr_dir2_threshold_500": 0.0075949890455728015
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
+      "scr_dir1_threshold_2": 0.2519684780685472,
+      "scr_metric_threshold_2": 0.1780415435515382,
+      "scr_dir2_threshold_2": 0.1780415435515382,
+      "scr_dir1_threshold_5": -0.007873912274188802,
+      "scr_metric_threshold_5": 0.249258302466854,
+      "scr_dir2_threshold_5": 0.249258302466854,
+      "scr_dir1_threshold_10": -0.04724394297291932,
+      "scr_metric_threshold_10": 0.2997032856130665,
+      "scr_dir2_threshold_10": 0.2997032856130665,
+      "scr_dir1_threshold_20": -0.16535403506911087,
+      "scr_metric_threshold_20": 0.28486650505613625,
+      "scr_dir2_threshold_20": 0.28486650505613625,
+      "scr_dir1_threshold_50": 0.4724408377125527,
+      "scr_metric_threshold_50": 0.3531157663653655,
+      "scr_dir2_threshold_50": 0.3531157663653655,
+      "scr_dir1_threshold_100": 0.5354330745616361,
+      "scr_metric_threshold_100": 0.10682496150459805,
+      "scr_dir2_threshold_100": 0.10682496150459805,
+      "scr_dir1_threshold_500": 0.33070853946600826,
+      "scr_metric_threshold_500": 0.20474778392768772,
+      "scr_dir2_threshold_500": 0.20474778392768772
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
+      "scr_dir1_threshold_2": 0.010869424374669583,
+      "scr_metric_threshold_2": 0.10980391240220748,
+      "scr_dir2_threshold_2": 0.10980391240220748,
+      "scr_dir1_threshold_5": 0.03260859706226871,
+      "scr_metric_threshold_5": 0.262744992625386,
+      "scr_dir2_threshold_5": 0.262744992625386,
+      "scr_dir1_threshold_10": 0.04347802143693829,
+      "scr_metric_threshold_10": 0.5333332554187635,
+      "scr_dir2_threshold_10": 0.5333332554187635,
+      "scr_dir1_threshold_20": 0.03260859706226871,
+      "scr_metric_threshold_20": 0.6156863065922739,
+      "scr_dir2_threshold_20": 0.6156863065922739,
+      "scr_dir1_threshold_50": -0.04347834537519825,
+      "scr_metric_threshold_50": 0.7098039124022075,
+      "scr_dir2_threshold_50": 0.7098039124022075,
+      "scr_dir1_threshold_100": -0.03804363318786346,
+      "scr_metric_threshold_100": 0.7686273868154523,
+      "scr_dir2_threshold_100": 0.7686273868154523,
+      "scr_dir1_threshold_500": 0.059782481937202626,
+      "scr_metric_threshold_500": 0.6549019562011037,
+      "scr_dir2_threshold_500": 0.6549019562011037
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
+      "scr_dir1_threshold_2": 0.09278322958848217,
+      "scr_metric_threshold_2": 0.0927419335456348,
+      "scr_dir2_threshold_2": 0.0927419335456348,
+      "scr_dir1_threshold_5": 0.12886585268435963,
+      "scr_metric_threshold_5": 0.1491935348194436,
+      "scr_dir2_threshold_5": 0.1491935348194436,
+      "scr_dir1_threshold_10": 0.18041219375810347,
+      "scr_metric_threshold_10": 0.16935480382142643,
+      "scr_dir2_threshold_10": 0.16935480382142643,
+      "scr_dir1_threshold_20": 0.25773170536871925,
+      "scr_metric_threshold_20": 0.20967734182539205,
+      "scr_dir2_threshold_20": 0.20967734182539205,
+      "scr_dir1_threshold_50": 0.3247420716607574,
+      "scr_metric_threshold_50": 0.41935492399206875,
+      "scr_dir2_threshold_50": 0.41935492399206875,
+      "scr_dir1_threshold_100": 0.26288658526843595,
+      "scr_metric_threshold_100": 0.4475806044583308,
+      "scr_dir2_threshold_100": 0.4475806044583308,
+      "scr_dir1_threshold_500": 0.2938143284645967,
+      "scr_metric_threshold_500": 0.6330644715496004,
+      "scr_dir2_threshold_500": 0.6330644715496004
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
+      "scr_dir1_threshold_2": 0.05855860935384807,
+      "scr_metric_threshold_2": 0.32888885827712394,
+      "scr_dir2_threshold_2": 0.32888885827712394,
+      "scr_dir1_threshold_5": 0.09009000301245093,
+      "scr_metric_threshold_5": 0.43111107814459504,
+      "scr_dir2_threshold_5": 0.43111107814459504,
+      "scr_dir1_threshold_10": 0.13513500451867638,
+      "scr_metric_threshold_10": 0.5244445245059836,
+      "scr_dir2_threshold_10": 0.5244445245059836,
+      "scr_dir1_threshold_20": 0.2432433303208824,
+      "scr_metric_threshold_20": 0.6088889324517851,
+      "scr_dir2_threshold_20": 0.6088889324517851,
+      "scr_dir1_threshold_50": 0.37387388838681374,
+      "scr_metric_threshold_50": 0.6666667549698347,
+      "scr_dir2_threshold_50": 0.6666667549698347,
+      "scr_dir1_threshold_100": 0.009008892905490125,
+      "scr_metric_threshold_100": 0.7022221139036694,
+      "scr_dir2_threshold_100": 0.7022221139036694,
+      "scr_dir1_threshold_500": 0.09009000301245093,
+      "scr_metric_threshold_500": 0.7555555496686777,
+      "scr_dir2_threshold_500": 0.7555555496686777
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
+      "scr_dir1_threshold_2": 0.03433482762748996,
+      "scr_metric_threshold_2": 0.03433482762748996,
+      "scr_dir2_threshold_2": 0.09523798711177515,
+      "scr_dir1_threshold_5": 0.060085820441166386,
+      "scr_metric_threshold_5": 0.060085820441166386,
+      "scr_dir2_threshold_5": 0.13809513807839202,
+      "scr_dir1_threshold_10": 0.03433482762748996,
+      "scr_metric_threshold_10": 0.03433482762748996,
+      "scr_dir2_threshold_10": 0.21428575483308432,
+      "scr_dir1_threshold_20": 0.10300422706858779,
+      "scr_metric_threshold_20": 0.10300422706858779,
+      "scr_dir2_threshold_20": 0.22857128260089646,
+      "scr_dir1_threshold_50": 0.17167388232356773,
+      "scr_metric_threshold_50": 0.17167388232356773,
+      "scr_dir2_threshold_50": 0.25238077937884024,
+      "scr_dir1_threshold_100": 0.10729601656855352,
+      "scr_metric_threshold_100": 0.10729601656855352,
+      "scr_dir2_threshold_100": 0.300000056766318,
+      "scr_dir1_threshold_500": 0.03862661712745569,
+      "scr_metric_threshold_500": 0.03862661712745569,
+      "scr_dir2_threshold_500": 0.3857143586995518
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_3",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

random_seed_eval_results/scr/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_4_custom_sae_eval_results.json

@@ -0,0 +1,323 @@
+{
+  "eval_type_id": "scr",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": true,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "34ee72b9-1763-4633-b984-223c06ba4460",
+  "datetime_epoch_millis": 1738792841644,
+  "eval_result_metrics": {
+    "scr_metrics": {
+      "scr_dir1_threshold_2": 0.23138581033127065,
+      "scr_metric_threshold_2": 0.13280867947941138,
+      "scr_dir2_threshold_2": 0.1354757659784241,
+      "scr_dir1_threshold_5": 0.25065267944291175,
+      "scr_metric_threshold_5": 0.19412669695057988,
+      "scr_dir2_threshold_5": 0.201731934928515,
+      "scr_dir1_threshold_10": 0.26031112509300863,
+      "scr_metric_threshold_10": 0.27721479325569365,
+      "scr_dir2_threshold_10": 0.2875535313264051,
+      "scr_dir1_threshold_20": 0.22049703318560473,
+      "scr_metric_threshold_20": 0.3181290025820012,
+      "scr_dir2_threshold_20": 0.3235142432818539,
+      "scr_dir1_threshold_50": 0.3025873216306098,
+      "scr_metric_threshold_50": 0.3842515908876025,
+      "scr_dir2_threshold_50": 0.3905181916365426,
+      "scr_dir1_threshold_100": 0.16621406066629962,
+      "scr_metric_threshold_100": 0.32000395023975886,
+      "scr_dir2_threshold_100": 0.3322893742558974,
+      "scr_dir1_threshold_500": -0.4676586781273777,
+      "scr_metric_threshold_500": 0.3251383965129933,
+      "scr_dir2_threshold_500": 0.3427886213575595
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_professor_nurse_results",
+      "scr_dir1_threshold_2": 0.4531256111803394,
+      "scr_metric_threshold_2": 0.022167494916722333,
+      "scr_dir2_threshold_2": 0.022167494916722333,
+      "scr_dir1_threshold_5": 0.5156247962732202,
+      "scr_metric_threshold_5": 0.044334989833444666,
+      "scr_dir2_threshold_5": 0.044334989833444666,
+      "scr_dir1_threshold_10": 0.4843752037267798,
+      "scr_metric_threshold_10": 0.05418707929913162,
+      "scr_dir2_threshold_10": 0.05418707929913162,
+      "scr_dir1_threshold_20": 0.43749988358469727,
+      "scr_metric_threshold_20": 0.07389155184943222,
+      "scr_dir2_threshold_20": 0.07389155184943222,
+      "scr_dir1_threshold_50": 0.42187508731147705,
+      "scr_metric_threshold_50": 0.12561575559160543,
+      "scr_dir2_threshold_50": 0.12561575559160543,
+      "scr_dir1_threshold_100": 0.42187508731147705,
+      "scr_metric_threshold_100": 0.19211824034177244,
+      "scr_dir2_threshold_100": 0.19211824034177244,
+      "scr_dir1_threshold_500": -1.3749997671693945,
+      "scr_metric_threshold_500": 0.35960581659093743,
+      "scr_dir2_threshold_500": 0.35960581659093743
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_architect_journalist_results",
+      "scr_dir1_threshold_2": 0.2803735818828607,
+      "scr_metric_threshold_2": 0.24495688024825762,
+      "scr_dir2_threshold_2": 0.24495688024825762,
+      "scr_dir1_threshold_5": 0.2803735818828607,
+      "scr_metric_threshold_5": 0.2997119051602506,
+      "scr_dir2_threshold_5": 0.2997119051602506,
+      "scr_dir1_threshold_10": 0.33644874390166457,
+      "scr_metric_threshold_10": 0.37175794536779155,
+      "scr_dir2_threshold_10": 0.37175794536779155,
+      "scr_dir1_threshold_20": 0.2149533022646424,
+      "scr_metric_threshold_20": 0.4293947775338243,
+      "scr_dir2_threshold_20": 0.4293947775338243,
+      "scr_dir1_threshold_50": 0.2149533022646424,
+      "scr_metric_threshold_50": 0.38328534615525994,
+      "scr_dir2_threshold_50": 0.38328534615525994,
+      "scr_dir1_threshold_100": -0.6448599067939272,
+      "scr_metric_threshold_100": -0.05475502491199297,
+      "scr_dir2_threshold_100": -0.05475502491199297,
+      "scr_dir1_threshold_500": -1.4018695805726726,
+      "scr_metric_threshold_500": -0.1527376739485105,
+      "scr_dir2_threshold_500": -0.1527376739485105
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_surgeon_psychologist_results",
+      "scr_dir1_threshold_2": 0.5468753201420825,
+      "scr_metric_threshold_2": 0.037974794330044616,
+      "scr_dir2_threshold_2": 0.037974794330044616,
+      "scr_dir1_threshold_5": 0.5625001164153027,
+      "scr_metric_threshold_5": 0.06075961056894363,
+      "scr_dir2_threshold_5": 0.06075961056894363,
+      "scr_dir1_threshold_10": 0.5625001164153027,
+      "scr_metric_threshold_10": 0.10632924304674166,
+      "scr_dir2_threshold_10": 0.10632924304674166,
+      "scr_dir1_threshold_20": 0.43749988358469727,
+      "scr_metric_threshold_20": 0.14936711191080249,
+      "scr_dir2_threshold_20": 0.14936711191080249,
+      "scr_dir1_threshold_50": 0.42187508731147705,
+      "scr_metric_threshold_50": 0.24303798958231634,
+      "scr_dir2_threshold_50": 0.24303798958231634,
+      "scr_dir1_threshold_100": 0.2656252619344312,
+      "scr_metric_threshold_100": 0.2860760093441966,
+      "scr_dir2_threshold_100": 0.2860760093441966,
+      "scr_dir1_threshold_500": -2.1718746216502662,
+      "scr_metric_threshold_500": 0.02784819256855401,
+      "scr_dir2_threshold_500": 0.02784819256855401
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_scr_attorney_teacher_results",
+      "scr_dir1_threshold_2": 0.2519684780685472,
+      "scr_metric_threshold_2": 0.16913958133840545,
+      "scr_dir2_threshold_2": 0.16913958133840545,
+      "scr_dir1_threshold_5": 0.06299223684908342,
+      "scr_metric_threshold_5": 0.25816026467998676,
+      "scr_dir2_threshold_5": 0.25816026467998676,
+      "scr_dir1_threshold_10": -0.03937003069873052,
+      "scr_metric_threshold_10": 0.32047488451379413,
+      "scr_dir2_threshold_10": 0.32047488451379413,
+      "scr_dir1_threshold_20": -0.18897624121946377,
+      "scr_metric_threshold_20": 0.2818991843184253,
+      "scr_dir2_threshold_20": 0.2818991843184253,
+      "scr_dir1_threshold_50": 0.46456692543836386,
+      "scr_metric_threshold_50": 0.3531157663653655,
+      "scr_dir2_threshold_50": 0.3531157663653655,
+      "scr_dir1_threshold_100": 0.5826770175345555,
+      "scr_metric_threshold_100": 0.10385764076688712,
+      "scr_dir2_threshold_100": 0.10385764076688712,
+      "scr_dir1_threshold_500": 0.4488191008899863,
+      "scr_metric_threshold_500": 0.19584582171455495,
+      "scr_dir2_threshold_500": 0.19584582171455495
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Books_CDs_and_Vinyl_results",
+      "scr_dir1_threshold_2": 0.08695636681213655,
+      "scr_metric_threshold_2": 0.13333325541876354,
+      "scr_dir2_threshold_2": 0.13333325541876354,
+      "scr_dir1_threshold_5": 0.08152165462480175,
+      "scr_metric_threshold_5": 0.24705868603311212,
+      "scr_dir2_threshold_5": 0.24705868603311212,
+      "scr_dir1_threshold_10": 0.08695636681213655,
+      "scr_metric_threshold_10": 0.5411765255867551,
+      "scr_dir2_threshold_10": 0.5411765255867551,
+      "scr_dir1_threshold_20": 0.09782611512506609,
+      "scr_metric_threshold_20": 0.6470586860331121,
+      "scr_dir2_threshold_20": 0.6470586860331121,
+      "scr_dir1_threshold_50": -0.05434776974986783,
+      "scr_metric_threshold_50": 0.7176469488264897,
+      "scr_dir2_threshold_50": 0.7176469488264897,
+      "scr_dir1_threshold_100": -0.05434776974986783,
+      "scr_metric_threshold_100": 0.7372547736309046,
+      "scr_dir2_threshold_100": 0.7372547736309046,
+      "scr_dir1_threshold_500": 0.05434776974986783,
+      "scr_metric_threshold_500": 0.7450980437988962,
+      "scr_dir2_threshold_500": 0.7450980437988962
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Software_Electronics_results",
+      "scr_dir1_threshold_2": 0.07731951161061577,
+      "scr_metric_threshold_2": 0.0927419335456348,
+      "scr_dir2_threshold_2": 0.0927419335456348,
+      "scr_dir1_threshold_5": 0.190721646317109,
+      "scr_metric_threshold_5": 0.16129039235714715,
+      "scr_dir2_threshold_5": 0.16129039235714715,
+      "scr_dir1_threshold_10": 0.24226798739085284,
+      "scr_metric_threshold_10": 0.20967734182539205,
+      "scr_dir2_threshold_10": 0.20967734182539205,
+      "scr_dir1_threshold_20": 0.3144329263421798,
+      "scr_metric_threshold_20": 0.20967734182539205,
+      "scr_dir2_threshold_20": 0.20967734182539205,
+      "scr_dir1_threshold_50": 0.28350487590559115,
+      "scr_metric_threshold_50": 0.3709677341825392,
+      "scr_dir2_threshold_50": 0.3709677341825392,
+      "scr_dir1_threshold_100": 0.309278353682891,
+      "scr_metric_threshold_100": 0.3870967974523824,
+      "scr_dir2_threshold_100": 0.3870967974523824,
+      "scr_dir1_threshold_500": 0.36082469475663487,
+      "scr_metric_threshold_500": 0.608870996815478,
+      "scr_dir2_threshold_500": 0.608870996815478
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Pet_Supplies_Office_Products_results",
+      "scr_dir1_threshold_2": 0.09009000301245093,
+      "scr_metric_threshold_2": 0.29777788609633066,
+      "scr_dir2_threshold_2": 0.29777788609633066,
+      "scr_dir1_threshold_5": 0.23423416892600485,
+      "scr_metric_threshold_5": 0.404444492716843,
+      "scr_dir2_threshold_5": 0.404444492716843,
+      "scr_dir1_threshold_10": 0.3063063861274755,
+      "scr_metric_threshold_10": 0.5111110993373553,
+      "scr_dir2_threshold_10": 0.5111110993373553,
+      "scr_dir1_threshold_20": 0.27477472397948527,
+      "scr_metric_threshold_20": 0.5777776953614876,
+      "scr_dir2_threshold_20": 0.5777776953614876,
+      "scr_dir1_threshold_50": 0.42792778279852933,
+      "scr_metric_threshold_50": 0.6399999046325785,
+      "scr_dir2_threshold_50": 0.6399999046325785,
+      "scr_dir1_threshold_100": 0.24774777677362747,
+      "scr_metric_threshold_100": 0.7066665006567107,
+      "scr_dir2_threshold_100": 0.7066665006567107,
+      "scr_dir1_threshold_500": 0.18468472096703434,
+      "scr_metric_threshold_500": 0.6577777165542479,
+      "scr_dir2_threshold_500": 0.6577777165542479
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_scr_Industrial_and_Scientific_Toys_and_Games_results",
+      "scr_dir1_threshold_2": 0.0643776099411321,
+      "scr_metric_threshold_2": 0.0643776099411321,
+      "scr_dir2_threshold_2": 0.08571430193323373,
+      "scr_dir1_threshold_5": 0.07725323425491137,
+      "scr_metric_threshold_5": 0.07725323425491137,
+      "scr_dir2_threshold_5": 0.13809513807839202,
+      "scr_dir1_threshold_10": 0.10300422706858779,
+      "scr_metric_threshold_10": 0.10300422706858779,
+      "scr_dir2_threshold_10": 0.18571413163427958,
+      "scr_dir1_threshold_20": 0.17596567182353345,
+      "scr_metric_threshold_20": 0.17596567182353345,
+      "scr_dir2_threshold_20": 0.21904759742235502,
+      "scr_dir1_threshold_50": 0.24034328176466555,
+      "scr_metric_threshold_50": 0.24034328176466555,
+      "scr_dir2_threshold_50": 0.2904760877561864,
+      "scr_dir1_threshold_100": 0.20171666463720986,
+      "scr_metric_threshold_100": 0.20171666463720986,
+      "scr_dir2_threshold_100": 0.300000056766318,
+      "scr_dir1_threshold_500": 0.15879825800978847,
+      "scr_metric_threshold_500": 0.15879825800978847,
+      "scr_dir2_threshold_500": 0.300000056766318
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_4",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

random_seed_eval_results/sparse_probing/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_0_custom_sae_eval_results.json

@@ -0,0 +1,670 @@
+{
+  "eval_type_id": "sparse_probing",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "LabHC/bias_in_bios_class_set2",
+      "LabHC/bias_in_bios_class_set3",
+      "canrager/amazon_reviews_mcauley_1and5",
+      "canrager/amazon_reviews_mcauley_1and5_sentiment",
+      "codeparrot/github-code",
+      "fancyzhx/ag_news",
+      "Helsinki-NLP/europarl"
+    ],
+    "probe_train_set_size": 4000,
+    "probe_test_set_size": 1000,
+    "context_length": 128,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "model_name": "gemma-2-2b",
+    "k_values": [
+      1,
+      2,
+      5
+    ],
+    "lower_vram_usage": false
+  },
+  "eval_id": "6b237941-8790-4424-841f-44d2b2d4b18c",
+  "datetime_epoch_millis": 1738794566740,
+  "eval_result_metrics": {
+    "llm": {
+      "llm_test_accuracy": 0.9582500416785479,
+      "llm_top_1_test_accuracy": 0.6746375,
+      "llm_top_2_test_accuracy": 0.7199437500000001,
+      "llm_top_5_test_accuracy": 0.78408125,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null
+    },
+    "sae": {
+      "sae_test_accuracy": 0.9553687926381826,
+      "sae_top_1_test_accuracy": 0.73944375,
+      "sae_top_2_test_accuracy": 0.7974,
+      "sae_top_5_test_accuracy": 0.8732937500000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_results",
+      "llm_test_accuracy": 0.9694000363349915,
+      "llm_top_1_test_accuracy": 0.6436000000000001,
+      "llm_top_2_test_accuracy": 0.6874,
+      "llm_top_5_test_accuracy": 0.7908,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9626000404357911,
+      "sae_top_1_test_accuracy": 0.7674,
+      "sae_top_2_test_accuracy": 0.842,
+      "sae_top_5_test_accuracy": 0.8918000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set2_results",
+      "llm_test_accuracy": 0.9524000525474549,
+      "llm_top_1_test_accuracy": 0.6764,
+      "llm_top_2_test_accuracy": 0.7150000000000001,
+      "llm_top_5_test_accuracy": 0.7592000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9502000451087952,
+      "sae_top_1_test_accuracy": 0.6858000000000001,
+      "sae_top_2_test_accuracy": 0.7654,
+      "sae_top_5_test_accuracy": 0.8273999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set3_results",
+      "llm_test_accuracy": 0.9290000438690186,
+      "llm_top_1_test_accuracy": 0.6864,
+      "llm_top_2_test_accuracy": 0.7316,
+      "llm_top_5_test_accuracy": 0.7666000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9254000306129455,
+      "sae_top_1_test_accuracy": 0.746,
+      "sae_top_2_test_accuracy": 0.8208,
+      "sae_top_5_test_accuracy": 0.8624,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_results",
+      "llm_test_accuracy": 0.916200053691864,
+      "llm_top_1_test_accuracy": 0.6113999999999999,
+      "llm_top_2_test_accuracy": 0.6481999999999999,
+      "llm_top_5_test_accuracy": 0.6894,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.915000057220459,
+      "sae_top_1_test_accuracy": 0.6953999999999999,
+      "sae_top_2_test_accuracy": 0.7772000000000001,
+      "sae_top_5_test_accuracy": 0.826,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_sentiment_results",
+      "llm_test_accuracy": 0.9820000529289246,
+      "llm_top_1_test_accuracy": 0.672,
+      "llm_top_2_test_accuracy": 0.724,
+      "llm_top_5_test_accuracy": 0.766,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9730000495910645,
+      "sae_top_1_test_accuracy": 0.832,
+      "sae_top_2_test_accuracy": 0.83,
+      "sae_top_5_test_accuracy": 0.948,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "codeparrot/github-code_results",
+      "llm_test_accuracy": 0.9714000344276428,
+      "llm_top_1_test_accuracy": 0.6452000000000001,
+      "llm_top_2_test_accuracy": 0.6928,
+      "llm_top_5_test_accuracy": 0.7726,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9684000372886657,
+      "sae_top_1_test_accuracy": 0.6340000000000001,
+      "sae_top_2_test_accuracy": 0.7074,
+      "sae_top_5_test_accuracy": 0.8161999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "fancyzhx/ag_news_results",
+      "llm_test_accuracy": 0.9460000544786453,
+      "llm_top_1_test_accuracy": 0.7325,
+      "llm_top_2_test_accuracy": 0.77375,
+      "llm_top_5_test_accuracy": 0.82125,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9497500509023666,
+      "sae_top_1_test_accuracy": 0.68075,
+      "sae_top_2_test_accuracy": 0.737,
+      "sae_top_5_test_accuracy": 0.82075,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "Helsinki-NLP/europarl_results",
+      "llm_test_accuracy": 0.9996000051498413,
+      "llm_top_1_test_accuracy": 0.7296,
+      "llm_top_2_test_accuracy": 0.7868,
+      "llm_top_5_test_accuracy": 0.9067999999999999,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9986000299453736,
+      "sae_top_1_test_accuracy": 0.8742000000000001,
+      "sae_top_2_test_accuracy": 0.8994,
+      "sae_top_5_test_accuracy": 0.9938,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_0",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1_results": {
+      "sae_test_accuracy": {
+        "0": 0.9440000653266907,
+        "1": 0.9600000381469727,
+        "2": 0.9520000219345093,
+        "6": 0.9820000529289246,
+        "9": 0.9750000238418579
+      },
+      "llm_test_accuracy": {
+        "0": 0.9510000348091125,
+        "1": 0.9670000672340393,
+        "2": 0.9520000219345093,
+        "6": 0.9930000305175781,
+        "9": 0.984000027179718
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.568,
+        "1": 0.629,
+        "2": 0.679,
+        "6": 0.791,
+        "9": 0.551
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.585,
+        "1": 0.666,
+        "2": 0.673,
+        "6": 0.801,
+        "9": 0.712
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.72,
+        "1": 0.707,
+        "2": 0.764,
+        "6": 0.899,
+        "9": 0.864
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.593,
+        "1": 0.615,
+        "2": 0.871,
+        "6": 0.833,
+        "9": 0.925
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.627,
+        "1": 0.799,
+        "2": 0.874,
+        "6": 0.981,
+        "9": 0.929
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.846,
+        "1": 0.81,
+        "2": 0.885,
+        "6": 0.981,
+        "9": 0.937
+      }
+    },
+    "LabHC/bias_in_bios_class_set2_results": {
+      "sae_test_accuracy": {
+        "11": 0.9650000333786011,
+        "13": 0.9470000267028809,
+        "14": 0.9540000557899475,
+        "18": 0.9260000586509705,
+        "19": 0.9590000510215759
+      },
+      "llm_test_accuracy": {
+        "11": 0.9620000720024109,
+        "13": 0.9470000267028809,
+        "14": 0.9580000638961792,
+        "18": 0.9310000538825989,
+        "19": 0.9640000462532043
+      },
+      "llm_top_1_test_accuracy": {
+        "11": 0.558,
+        "13": 0.673,
+        "14": 0.656,
+        "18": 0.702,
+        "19": 0.793
+      },
+      "llm_top_2_test_accuracy": {
+        "11": 0.686,
+        "13": 0.713,
+        "14": 0.687,
+        "18": 0.724,
+        "19": 0.765
+      },
+      "llm_top_5_test_accuracy": {
+        "11": 0.782,
+        "13": 0.742,
+        "14": 0.716,
+        "18": 0.725,
+        "19": 0.831
+      },
+      "sae_top_1_test_accuracy": {
+        "11": 0.596,
+        "13": 0.692,
+        "14": 0.648,
+        "18": 0.701,
+        "19": 0.792
+      },
+      "sae_top_2_test_accuracy": {
+        "11": 0.745,
+        "13": 0.681,
+        "14": 0.862,
+        "18": 0.698,
+        "19": 0.841
+      },
+      "sae_top_5_test_accuracy": {
+        "11": 0.95,
+        "13": 0.713,
+        "14": 0.88,
+        "18": 0.739,
+        "19": 0.855
+      }
+    },
+    "LabHC/bias_in_bios_class_set3_results": {
+      "sae_test_accuracy": {
+        "20": 0.9530000686645508,
+        "21": 0.9200000166893005,
+        "22": 0.906000018119812,
+        "25": 0.9520000219345093,
+        "26": 0.8960000276565552
+      },
+      "llm_test_accuracy": {
+        "20": 0.9610000252723694,
+        "21": 0.9140000343322754,
+        "22": 0.9170000553131104,
+        "25": 0.9630000591278076,
+        "26": 0.89000004529953
+      },
+      "llm_top_1_test_accuracy": {
+        "20": 0.711,
+        "21": 0.771,
+        "22": 0.637,
+        "25": 0.687,
+        "26": 0.626
+      },
+      "llm_top_2_test_accuracy": {
+        "20": 0.809,
+        "21": 0.764,
+        "22": 0.659,
+        "25": 0.766,
+        "26": 0.66
+      },
+      "llm_top_5_test_accuracy": {
+        "20": 0.858,
+        "21": 0.795,
+        "22": 0.715,
+        "25": 0.786,
+        "26": 0.679
+      },
+      "sae_top_1_test_accuracy": {
+        "20": 0.873,
+        "21": 0.521,
+        "22": 0.854,
+        "25": 0.884,
+        "26": 0.598
+      },
+      "sae_top_2_test_accuracy": {
+        "20": 0.886,
+        "21": 0.824,
+        "22": 0.88,
+        "25": 0.875,
+        "26": 0.639
+      },
+      "sae_top_5_test_accuracy": {
+        "20": 0.915,
+        "21": 0.843,
+        "22": 0.884,
+        "25": 0.894,
+        "26": 0.776
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_results": {
+      "sae_test_accuracy": {
+        "1": 0.9440000653266907,
+        "2": 0.9280000329017639,
+        "3": 0.9310000538825989,
+        "5": 0.9070000648498535,
+        "6": 0.8650000691413879
+      },
+      "llm_test_accuracy": {
+        "1": 0.9500000476837158,
+        "2": 0.937000036239624,
+        "3": 0.9260000586509705,
+        "5": 0.9120000600814819,
+        "6": 0.8560000658035278
+      },
+      "llm_top_1_test_accuracy": {
+        "1": 0.693,
+        "2": 0.607,
+        "3": 0.579,
+        "5": 0.577,
+        "6": 0.601
+      },
+      "llm_top_2_test_accuracy": {
+        "1": 0.747,
+        "2": 0.64,
+        "3": 0.607,
+        "5": 0.628,
+        "6": 0.619
+      },
+      "llm_top_5_test_accuracy": {
+        "1": 0.78,
+        "2": 0.657,
+        "3": 0.667,
+        "5": 0.659,
+        "6": 0.684
+      },
+      "sae_top_1_test_accuracy": {
+        "1": 0.871,
+        "2": 0.598,
+        "3": 0.56,
+        "5": 0.861,
+        "6": 0.587
+      },
+      "sae_top_2_test_accuracy": {
+        "1": 0.899,
+        "2": 0.822,
+        "3": 0.647,
+        "5": 0.873,
+        "6": 0.645
+      },
+      "sae_top_5_test_accuracy": {
+        "1": 0.929,
+        "2": 0.861,
+        "3": 0.756,
+        "5": 0.863,
+        "6": 0.721
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_sentiment_results": {
+      "sae_test_accuracy": {
+        "1.0": 0.9720000624656677,
+        "5.0": 0.9740000367164612
+      },
+      "llm_test_accuracy": {
+        "1.0": 0.9820000529289246,
+        "5.0": 0.9820000529289246
+      },
+      "llm_top_1_test_accuracy": {
+        "1.0": 0.672,
+        "5.0": 0.672
+      },
+      "llm_top_2_test_accuracy": {
+        "1.0": 0.724,
+        "5.0": 0.724
+      },
+      "llm_top_5_test_accuracy": {
+        "1.0": 0.766,
+        "5.0": 0.766
+      },
+      "sae_top_1_test_accuracy": {
+        "1.0": 0.832,
+        "5.0": 0.832
+      },
+      "sae_top_2_test_accuracy": {
+        "1.0": 0.83,
+        "5.0": 0.83
+      },
+      "sae_top_5_test_accuracy": {
+        "1.0": 0.948,
+        "5.0": 0.948
+      }
+    },
+    "codeparrot/github-code_results": {
+      "sae_test_accuracy": {
+        "C": 0.9540000557899475,
+        "Python": 0.984000027179718,
+        "HTML": 0.9800000190734863,
+        "Java": 0.9670000672340393,
+        "PHP": 0.9570000171661377
+      },
+      "llm_test_accuracy": {
+        "C": 0.956000030040741,
+        "Python": 0.987000048160553,
+        "HTML": 0.9940000176429749,
+        "Java": 0.9610000252723694,
+        "PHP": 0.9590000510215759
+      },
+      "llm_top_1_test_accuracy": {
+        "C": 0.657,
+        "Python": 0.636,
+        "HTML": 0.733,
+        "Java": 0.616,
+        "PHP": 0.584
+      },
+      "llm_top_2_test_accuracy": {
+        "C": 0.671,
+        "Python": 0.668,
+        "HTML": 0.803,
+        "Java": 0.68,
+        "PHP": 0.642
+      },
+      "llm_top_5_test_accuracy": {
+        "C": 0.765,
+        "Python": 0.727,
+        "HTML": 0.943,
+        "Java": 0.735,
+        "PHP": 0.693
+      },
+      "sae_top_1_test_accuracy": {
+        "C": 0.616,
+        "Python": 0.632,
+        "HTML": 0.699,
+        "Java": 0.631,
+        "PHP": 0.592
+      },
+      "sae_top_2_test_accuracy": {
+        "C": 0.62,
+        "Python": 0.923,
+        "HTML": 0.753,
+        "Java": 0.643,
+        "PHP": 0.598
+      },
+      "sae_top_5_test_accuracy": {
+        "C": 0.678,
+        "Python": 0.931,
+        "HTML": 0.882,
+        "Java": 0.663,
+        "PHP": 0.927
+      }
+    },
+    "fancyzhx/ag_news_results": {
+      "sae_test_accuracy": {
+        "0": 0.940000057220459,
+        "1": 0.987000048160553,
+        "2": 0.9250000715255737,
+        "3": 0.9470000267028809
+      },
+      "llm_test_accuracy": {
+        "0": 0.9390000700950623,
+        "1": 0.984000027179718,
+        "2": 0.9160000681877136,
+        "3": 0.9450000524520874
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.806,
+        "1": 0.662,
+        "2": 0.671,
+        "3": 0.791
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.796,
+        "1": 0.796,
+        "2": 0.694,
+        "3": 0.809
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.816,
+        "1": 0.885,
+        "2": 0.744,
+        "3": 0.84
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.725,
+        "1": 0.701,
+        "2": 0.675,
+        "3": 0.622
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.809,
+        "1": 0.69,
+        "2": 0.811,
+        "3": 0.638
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.842,
+        "1": 0.838,
+        "2": 0.837,
+        "3": 0.766
+      }
+    },
+    "Helsinki-NLP/europarl_results": {
+      "sae_test_accuracy": {
+        "en": 0.9980000257492065,
+        "fr": 1.0,
+        "de": 0.999000072479248,
+        "es": 0.9980000257492065,
+        "nl": 0.9980000257492065
+      },
+      "llm_test_accuracy": {
+        "en": 1.0,
+        "fr": 1.0,
+        "de": 1.0,
+        "es": 1.0,
+        "nl": 0.9980000257492065
+      },
+      "llm_top_1_test_accuracy": {
+        "en": 0.749,
+        "fr": 0.605,
+        "de": 0.741,
+        "es": 0.913,
+        "nl": 0.64
+      },
+      "llm_top_2_test_accuracy": {
+        "en": 0.831,
+        "fr": 0.607,
+        "de": 0.828,
+        "es": 0.915,
+        "nl": 0.753
+      },
+      "llm_top_5_test_accuracy": {
+        "en": 0.888,
+        "fr": 0.924,
+        "de": 0.882,
+        "es": 0.98,
+        "nl": 0.86
+      },
+      "sae_top_1_test_accuracy": {
+        "en": 0.838,
+        "fr": 0.992,
+        "de": 0.914,
+        "es": 0.87,
+        "nl": 0.757
+      },
+      "sae_top_2_test_accuracy": {
+        "en": 0.838,
+        "fr": 0.99,
+        "de": 0.925,
+        "es": 0.99,
+        "nl": 0.754
+      },
+      "sae_top_5_test_accuracy": {
+        "en": 0.998,
+        "fr": 0.994,
+        "de": 0.984,
+        "es": 0.995,
+        "nl": 0.998
+      }
+    }
+  }
+}

random_seed_eval_results/sparse_probing/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_1_custom_sae_eval_results.json

@@ -0,0 +1,670 @@
+{
+  "eval_type_id": "sparse_probing",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "LabHC/bias_in_bios_class_set2",
+      "LabHC/bias_in_bios_class_set3",
+      "canrager/amazon_reviews_mcauley_1and5",
+      "canrager/amazon_reviews_mcauley_1and5_sentiment",
+      "codeparrot/github-code",
+      "fancyzhx/ag_news",
+      "Helsinki-NLP/europarl"
+    ],
+    "probe_train_set_size": 4000,
+    "probe_test_set_size": 1000,
+    "context_length": 128,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "model_name": "gemma-2-2b",
+    "k_values": [
+      1,
+      2,
+      5
+    ],
+    "lower_vram_usage": false
+  },
+  "eval_id": "0e9c84dc-6835-48de-9a61-d54bcf48b3aa",
+  "datetime_epoch_millis": 1738794472536,
+  "eval_result_metrics": {
+    "llm": {
+      "llm_test_accuracy": 0.9582500416785479,
+      "llm_top_1_test_accuracy": 0.6746375,
+      "llm_top_2_test_accuracy": 0.7199437500000001,
+      "llm_top_5_test_accuracy": 0.78408125,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null
+    },
+    "sae": {
+      "sae_test_accuracy": 0.9562937960028648,
+      "sae_top_1_test_accuracy": 0.7659312499999998,
+      "sae_top_2_test_accuracy": 0.8051812500000001,
+      "sae_top_5_test_accuracy": 0.87295625,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_results",
+      "llm_test_accuracy": 0.9694000363349915,
+      "llm_top_1_test_accuracy": 0.6436000000000001,
+      "llm_top_2_test_accuracy": 0.6874,
+      "llm_top_5_test_accuracy": 0.7908,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9636000394821167,
+      "sae_top_1_test_accuracy": 0.767,
+      "sae_top_2_test_accuracy": 0.8443999999999999,
+      "sae_top_5_test_accuracy": 0.9029999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set2_results",
+      "llm_test_accuracy": 0.9524000525474549,
+      "llm_top_1_test_accuracy": 0.6764,
+      "llm_top_2_test_accuracy": 0.7150000000000001,
+      "llm_top_5_test_accuracy": 0.7592000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9478000402450562,
+      "sae_top_1_test_accuracy": 0.6898,
+      "sae_top_2_test_accuracy": 0.7598,
+      "sae_top_5_test_accuracy": 0.8517999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set3_results",
+      "llm_test_accuracy": 0.9290000438690186,
+      "llm_top_1_test_accuracy": 0.6864,
+      "llm_top_2_test_accuracy": 0.7316,
+      "llm_top_5_test_accuracy": 0.7666000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9302000403404236,
+      "sae_top_1_test_accuracy": 0.7866,
+      "sae_top_2_test_accuracy": 0.8008,
+      "sae_top_5_test_accuracy": 0.8625999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_results",
+      "llm_test_accuracy": 0.916200053691864,
+      "llm_top_1_test_accuracy": 0.6113999999999999,
+      "llm_top_2_test_accuracy": 0.6481999999999999,
+      "llm_top_5_test_accuracy": 0.6894,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9182000517845154,
+      "sae_top_1_test_accuracy": 0.7419999999999999,
+      "sae_top_2_test_accuracy": 0.8026,
+      "sae_top_5_test_accuracy": 0.8196,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_sentiment_results",
+      "llm_test_accuracy": 0.9820000529289246,
+      "llm_top_1_test_accuracy": 0.672,
+      "llm_top_2_test_accuracy": 0.724,
+      "llm_top_5_test_accuracy": 0.766,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9730000495910645,
+      "sae_top_1_test_accuracy": 0.913,
+      "sae_top_2_test_accuracy": 0.913,
+      "sae_top_5_test_accuracy": 0.942,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "codeparrot/github-code_results",
+      "llm_test_accuracy": 0.9714000344276428,
+      "llm_top_1_test_accuracy": 0.6452000000000001,
+      "llm_top_2_test_accuracy": 0.6928,
+      "llm_top_5_test_accuracy": 0.7726,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9676000356674195,
+      "sae_top_1_test_accuracy": 0.6417999999999999,
+      "sae_top_2_test_accuracy": 0.6686000000000001,
+      "sae_top_5_test_accuracy": 0.8206,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "fancyzhx/ag_news_results",
+      "llm_test_accuracy": 0.9460000544786453,
+      "llm_top_1_test_accuracy": 0.7325,
+      "llm_top_2_test_accuracy": 0.77375,
+      "llm_top_5_test_accuracy": 0.82125,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9507500529289246,
+      "sae_top_1_test_accuracy": 0.7132499999999999,
+      "sae_top_2_test_accuracy": 0.73725,
+      "sae_top_5_test_accuracy": 0.78725,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "Helsinki-NLP/europarl_results",
+      "llm_test_accuracy": 0.9996000051498413,
+      "llm_top_1_test_accuracy": 0.7296,
+      "llm_top_2_test_accuracy": 0.7868,
+      "llm_top_5_test_accuracy": 0.9067999999999999,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9992000579833984,
+      "sae_top_1_test_accuracy": 0.874,
+      "sae_top_2_test_accuracy": 0.915,
+      "sae_top_5_test_accuracy": 0.9968,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_1",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1_results": {
+      "sae_test_accuracy": {
+        "0": 0.9520000219345093,
+        "1": 0.9610000252723694,
+        "2": 0.9450000524520874,
+        "6": 0.9860000610351562,
+        "9": 0.9740000367164612
+      },
+      "llm_test_accuracy": {
+        "0": 0.9510000348091125,
+        "1": 0.9670000672340393,
+        "2": 0.9520000219345093,
+        "6": 0.9930000305175781,
+        "9": 0.984000027179718
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.568,
+        "1": 0.629,
+        "2": 0.679,
+        "6": 0.791,
+        "9": 0.551
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.585,
+        "1": 0.666,
+        "2": 0.673,
+        "6": 0.801,
+        "9": 0.712
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.72,
+        "1": 0.707,
+        "2": 0.764,
+        "6": 0.899,
+        "9": 0.864
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.58,
+        "1": 0.632,
+        "2": 0.862,
+        "6": 0.825,
+        "9": 0.936
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.619,
+        "1": 0.814,
+        "2": 0.88,
+        "6": 0.982,
+        "9": 0.927
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.856,
+        "1": 0.85,
+        "2": 0.876,
+        "6": 0.982,
+        "9": 0.951
+      }
+    },
+    "LabHC/bias_in_bios_class_set2_results": {
+      "sae_test_accuracy": {
+        "11": 0.956000030040741,
+        "13": 0.9490000605583191,
+        "14": 0.9500000476837158,
+        "18": 0.9190000295639038,
+        "19": 0.9650000333786011
+      },
+      "llm_test_accuracy": {
+        "11": 0.9620000720024109,
+        "13": 0.9470000267028809,
+        "14": 0.9580000638961792,
+        "18": 0.9310000538825989,
+        "19": 0.9640000462532043
+      },
+      "llm_top_1_test_accuracy": {
+        "11": 0.558,
+        "13": 0.673,
+        "14": 0.656,
+        "18": 0.702,
+        "19": 0.793
+      },
+      "llm_top_2_test_accuracy": {
+        "11": 0.686,
+        "13": 0.713,
+        "14": 0.687,
+        "18": 0.724,
+        "19": 0.765
+      },
+      "llm_top_5_test_accuracy": {
+        "11": 0.782,
+        "13": 0.742,
+        "14": 0.716,
+        "18": 0.725,
+        "19": 0.831
+      },
+      "sae_top_1_test_accuracy": {
+        "11": 0.59,
+        "13": 0.684,
+        "14": 0.637,
+        "18": 0.692,
+        "19": 0.846
+      },
+      "sae_top_2_test_accuracy": {
+        "11": 0.735,
+        "13": 0.657,
+        "14": 0.878,
+        "18": 0.683,
+        "19": 0.846
+      },
+      "sae_top_5_test_accuracy": {
+        "11": 0.949,
+        "13": 0.69,
+        "14": 0.881,
+        "18": 0.892,
+        "19": 0.847
+      }
+    },
+    "LabHC/bias_in_bios_class_set3_results": {
+      "sae_test_accuracy": {
+        "20": 0.9590000510215759,
+        "21": 0.9340000152587891,
+        "22": 0.9120000600814819,
+        "25": 0.9610000252723694,
+        "26": 0.8850000500679016
+      },
+      "llm_test_accuracy": {
+        "20": 0.9610000252723694,
+        "21": 0.9140000343322754,
+        "22": 0.9170000553131104,
+        "25": 0.9630000591278076,
+        "26": 0.89000004529953
+      },
+      "llm_top_1_test_accuracy": {
+        "20": 0.711,
+        "21": 0.771,
+        "22": 0.637,
+        "25": 0.687,
+        "26": 0.626
+      },
+      "llm_top_2_test_accuracy": {
+        "20": 0.809,
+        "21": 0.764,
+        "22": 0.659,
+        "25": 0.766,
+        "26": 0.66
+      },
+      "llm_top_5_test_accuracy": {
+        "20": 0.858,
+        "21": 0.795,
+        "22": 0.715,
+        "25": 0.786,
+        "26": 0.679
+      },
+      "sae_top_1_test_accuracy": {
+        "20": 0.874,
+        "21": 0.763,
+        "22": 0.817,
+        "25": 0.883,
+        "26": 0.596
+      },
+      "sae_top_2_test_accuracy": {
+        "20": 0.9,
+        "21": 0.758,
+        "22": 0.859,
+        "25": 0.863,
+        "26": 0.624
+      },
+      "sae_top_5_test_accuracy": {
+        "20": 0.93,
+        "21": 0.846,
+        "22": 0.858,
+        "25": 0.887,
+        "26": 0.792
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_results": {
+      "sae_test_accuracy": {
+        "1": 0.9440000653266907,
+        "2": 0.9320000410079956,
+        "3": 0.9170000553131104,
+        "5": 0.9250000715255737,
+        "6": 0.8730000257492065
+      },
+      "llm_test_accuracy": {
+        "1": 0.9500000476837158,
+        "2": 0.937000036239624,
+        "3": 0.9260000586509705,
+        "5": 0.9120000600814819,
+        "6": 0.8560000658035278
+      },
+      "llm_top_1_test_accuracy": {
+        "1": 0.693,
+        "2": 0.607,
+        "3": 0.579,
+        "5": 0.577,
+        "6": 0.601
+      },
+      "llm_top_2_test_accuracy": {
+        "1": 0.747,
+        "2": 0.64,
+        "3": 0.607,
+        "5": 0.628,
+        "6": 0.619
+      },
+      "llm_top_5_test_accuracy": {
+        "1": 0.78,
+        "2": 0.657,
+        "3": 0.667,
+        "5": 0.659,
+        "6": 0.684
+      },
+      "sae_top_1_test_accuracy": {
+        "1": 0.811,
+        "2": 0.867,
+        "3": 0.578,
+        "5": 0.86,
+        "6": 0.594
+      },
+      "sae_top_2_test_accuracy": {
+        "1": 0.89,
+        "2": 0.876,
+        "3": 0.675,
+        "5": 0.87,
+        "6": 0.702
+      },
+      "sae_top_5_test_accuracy": {
+        "1": 0.911,
+        "2": 0.874,
+        "3": 0.709,
+        "5": 0.879,
+        "6": 0.725
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_sentiment_results": {
+      "sae_test_accuracy": {
+        "1.0": 0.9720000624656677,
+        "5.0": 0.9740000367164612
+      },
+      "llm_test_accuracy": {
+        "1.0": 0.9820000529289246,
+        "5.0": 0.9820000529289246
+      },
+      "llm_top_1_test_accuracy": {
+        "1.0": 0.672,
+        "5.0": 0.672
+      },
+      "llm_top_2_test_accuracy": {
+        "1.0": 0.724,
+        "5.0": 0.724
+      },
+      "llm_top_5_test_accuracy": {
+        "1.0": 0.766,
+        "5.0": 0.766
+      },
+      "sae_top_1_test_accuracy": {
+        "1.0": 0.913,
+        "5.0": 0.913
+      },
+      "sae_top_2_test_accuracy": {
+        "1.0": 0.913,
+        "5.0": 0.913
+      },
+      "sae_top_5_test_accuracy": {
+        "1.0": 0.942,
+        "5.0": 0.942
+      }
+    },
+    "codeparrot/github-code_results": {
+      "sae_test_accuracy": {
+        "C": 0.9430000185966492,
+        "Python": 0.987000048160553,
+        "HTML": 0.9910000562667847,
+        "Java": 0.9600000381469727,
+        "PHP": 0.9570000171661377
+      },
+      "llm_test_accuracy": {
+        "C": 0.956000030040741,
+        "Python": 0.987000048160553,
+        "HTML": 0.9940000176429749,
+        "Java": 0.9610000252723694,
+        "PHP": 0.9590000510215759
+      },
+      "llm_top_1_test_accuracy": {
+        "C": 0.657,
+        "Python": 0.636,
+        "HTML": 0.733,
+        "Java": 0.616,
+        "PHP": 0.584
+      },
+      "llm_top_2_test_accuracy": {
+        "C": 0.671,
+        "Python": 0.668,
+        "HTML": 0.803,
+        "Java": 0.68,
+        "PHP": 0.642
+      },
+      "llm_top_5_test_accuracy": {
+        "C": 0.765,
+        "Python": 0.727,
+        "HTML": 0.943,
+        "Java": 0.735,
+        "PHP": 0.693
+      },
+      "sae_top_1_test_accuracy": {
+        "C": 0.635,
+        "Python": 0.636,
+        "HTML": 0.686,
+        "Java": 0.643,
+        "PHP": 0.609
+      },
+      "sae_top_2_test_accuracy": {
+        "C": 0.657,
+        "Python": 0.66,
+        "HTML": 0.777,
+        "Java": 0.647,
+        "PHP": 0.602
+      },
+      "sae_top_5_test_accuracy": {
+        "C": 0.687,
+        "Python": 0.938,
+        "HTML": 0.896,
+        "Java": 0.656,
+        "PHP": 0.926
+      }
+    },
+    "fancyzhx/ag_news_results": {
+      "sae_test_accuracy": {
+        "0": 0.9450000524520874,
+        "1": 0.9860000610351562,
+        "2": 0.9250000715255737,
+        "3": 0.9470000267028809
+      },
+      "llm_test_accuracy": {
+        "0": 0.9390000700950623,
+        "1": 0.984000027179718,
+        "2": 0.9160000681877136,
+        "3": 0.9450000524520874
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.806,
+        "1": 0.662,
+        "2": 0.671,
+        "3": 0.791
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.796,
+        "1": 0.796,
+        "2": 0.694,
+        "3": 0.809
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.816,
+        "1": 0.885,
+        "2": 0.744,
+        "3": 0.84
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.799,
+        "1": 0.687,
+        "2": 0.724,
+        "3": 0.643
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.814,
+        "1": 0.675,
+        "2": 0.819,
+        "3": 0.641
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.849,
+        "1": 0.798,
+        "2": 0.811,
+        "3": 0.691
+      }
+    },
+    "Helsinki-NLP/europarl_results": {
+      "sae_test_accuracy": {
+        "en": 0.999000072479248,
+        "fr": 0.999000072479248,
+        "de": 1.0,
+        "es": 0.999000072479248,
+        "nl": 0.999000072479248
+      },
+      "llm_test_accuracy": {
+        "en": 1.0,
+        "fr": 1.0,
+        "de": 1.0,
+        "es": 1.0,
+        "nl": 0.9980000257492065
+      },
+      "llm_top_1_test_accuracy": {
+        "en": 0.749,
+        "fr": 0.605,
+        "de": 0.741,
+        "es": 0.913,
+        "nl": 0.64
+      },
+      "llm_top_2_test_accuracy": {
+        "en": 0.831,
+        "fr": 0.607,
+        "de": 0.828,
+        "es": 0.915,
+        "nl": 0.753
+      },
+      "llm_top_5_test_accuracy": {
+        "en": 0.888,
+        "fr": 0.924,
+        "de": 0.882,
+        "es": 0.98,
+        "nl": 0.86
+      },
+      "sae_top_1_test_accuracy": {
+        "en": 0.83,
+        "fr": 0.995,
+        "de": 0.905,
+        "es": 0.886,
+        "nl": 0.754
+      },
+      "sae_top_2_test_accuracy": {
+        "en": 0.856,
+        "fr": 0.997,
+        "de": 0.991,
+        "es": 0.991,
+        "nl": 0.74
+      },
+      "sae_top_5_test_accuracy": {
+        "en": 0.999,
+        "fr": 0.997,
+        "de": 0.994,
+        "es": 0.995,
+        "nl": 0.999
+      }
+    }
+  }
+}

random_seed_eval_results/sparse_probing/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_2_custom_sae_eval_results.json

@@ -0,0 +1,670 @@
+{
+  "eval_type_id": "sparse_probing",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "LabHC/bias_in_bios_class_set2",
+      "LabHC/bias_in_bios_class_set3",
+      "canrager/amazon_reviews_mcauley_1and5",
+      "canrager/amazon_reviews_mcauley_1and5_sentiment",
+      "codeparrot/github-code",
+      "fancyzhx/ag_news",
+      "Helsinki-NLP/europarl"
+    ],
+    "probe_train_set_size": 4000,
+    "probe_test_set_size": 1000,
+    "context_length": 128,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "model_name": "gemma-2-2b",
+    "k_values": [
+      1,
+      2,
+      5
+    ],
+    "lower_vram_usage": false
+  },
+  "eval_id": "44a6fcc3-3e44-4e32-aedf-75d4817b1efc",
+  "datetime_epoch_millis": 1738794660341,
+  "eval_result_metrics": {
+    "llm": {
+      "llm_test_accuracy": 0.9582500416785479,
+      "llm_top_1_test_accuracy": 0.6746375,
+      "llm_top_2_test_accuracy": 0.7199437500000001,
+      "llm_top_5_test_accuracy": 0.78408125,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null
+    },
+    "sae": {
+      "sae_test_accuracy": 0.9547500442713499,
+      "sae_top_1_test_accuracy": 0.7546875000000001,
+      "sae_top_2_test_accuracy": 0.8126875,
+      "sae_top_5_test_accuracy": 0.87828125,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_results",
+      "llm_test_accuracy": 0.9694000363349915,
+      "llm_top_1_test_accuracy": 0.6436000000000001,
+      "llm_top_2_test_accuracy": 0.6874,
+      "llm_top_5_test_accuracy": 0.7908,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9614000439643859,
+      "sae_top_1_test_accuracy": 0.7754000000000001,
+      "sae_top_2_test_accuracy": 0.8552,
+      "sae_top_5_test_accuracy": 0.9076000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set2_results",
+      "llm_test_accuracy": 0.9524000525474549,
+      "llm_top_1_test_accuracy": 0.6764,
+      "llm_top_2_test_accuracy": 0.7150000000000001,
+      "llm_top_5_test_accuracy": 0.7592000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9456000447273254,
+      "sae_top_1_test_accuracy": 0.6892,
+      "sae_top_2_test_accuracy": 0.7634000000000001,
+      "sae_top_5_test_accuracy": 0.8103999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set3_results",
+      "llm_test_accuracy": 0.9290000438690186,
+      "llm_top_1_test_accuracy": 0.6864,
+      "llm_top_2_test_accuracy": 0.7316,
+      "llm_top_5_test_accuracy": 0.7666000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9258000254631042,
+      "sae_top_1_test_accuracy": 0.7434,
+      "sae_top_2_test_accuracy": 0.7851999999999999,
+      "sae_top_5_test_accuracy": 0.8620000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_results",
+      "llm_test_accuracy": 0.916200053691864,
+      "llm_top_1_test_accuracy": 0.6113999999999999,
+      "llm_top_2_test_accuracy": 0.6481999999999999,
+      "llm_top_5_test_accuracy": 0.6894,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9164000391960144,
+      "sae_top_1_test_accuracy": 0.751,
+      "sae_top_2_test_accuracy": 0.7876000000000001,
+      "sae_top_5_test_accuracy": 0.8395999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_sentiment_results",
+      "llm_test_accuracy": 0.9820000529289246,
+      "llm_top_1_test_accuracy": 0.672,
+      "llm_top_2_test_accuracy": 0.724,
+      "llm_top_5_test_accuracy": 0.766,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9715000689029694,
+      "sae_top_1_test_accuracy": 0.89,
+      "sae_top_2_test_accuracy": 0.891,
+      "sae_top_5_test_accuracy": 0.936,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "codeparrot/github-code_results",
+      "llm_test_accuracy": 0.9714000344276428,
+      "llm_top_1_test_accuracy": 0.6452000000000001,
+      "llm_top_2_test_accuracy": 0.6928,
+      "llm_top_5_test_accuracy": 0.7726,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9690000414848328,
+      "sae_top_1_test_accuracy": 0.635,
+      "sae_top_2_test_accuracy": 0.7045999999999999,
+      "sae_top_5_test_accuracy": 0.8392000000000002,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "fancyzhx/ag_news_results",
+      "llm_test_accuracy": 0.9460000544786453,
+      "llm_top_1_test_accuracy": 0.7325,
+      "llm_top_2_test_accuracy": 0.77375,
+      "llm_top_5_test_accuracy": 0.82125,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9495000392198563,
+      "sae_top_1_test_accuracy": 0.6825,
+      "sae_top_2_test_accuracy": 0.7375,
+      "sae_top_5_test_accuracy": 0.8352499999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "Helsinki-NLP/europarl_results",
+      "llm_test_accuracy": 0.9996000051498413,
+      "llm_top_1_test_accuracy": 0.7296,
+      "llm_top_2_test_accuracy": 0.7868,
+      "llm_top_5_test_accuracy": 0.9067999999999999,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9988000512123107,
+      "sae_top_1_test_accuracy": 0.8710000000000001,
+      "sae_top_2_test_accuracy": 0.977,
+      "sae_top_5_test_accuracy": 0.9962,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_2",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1_results": {
+      "sae_test_accuracy": {
+        "0": 0.9450000524520874,
+        "1": 0.956000030040741,
+        "2": 0.9490000605583191,
+        "6": 0.9830000400543213,
+        "9": 0.9740000367164612
+      },
+      "llm_test_accuracy": {
+        "0": 0.9510000348091125,
+        "1": 0.9670000672340393,
+        "2": 0.9520000219345093,
+        "6": 0.9930000305175781,
+        "9": 0.984000027179718
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.568,
+        "1": 0.629,
+        "2": 0.679,
+        "6": 0.791,
+        "9": 0.551
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.585,
+        "1": 0.666,
+        "2": 0.673,
+        "6": 0.801,
+        "9": 0.712
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.72,
+        "1": 0.707,
+        "2": 0.764,
+        "6": 0.899,
+        "9": 0.864
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.606,
+        "1": 0.633,
+        "2": 0.878,
+        "6": 0.826,
+        "9": 0.934
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.662,
+        "1": 0.819,
+        "2": 0.885,
+        "6": 0.981,
+        "9": 0.929
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.867,
+        "1": 0.854,
+        "2": 0.89,
+        "6": 0.984,
+        "9": 0.943
+      }
+    },
+    "LabHC/bias_in_bios_class_set2_results": {
+      "sae_test_accuracy": {
+        "11": 0.9570000171661377,
+        "13": 0.940000057220459,
+        "14": 0.9530000686645508,
+        "18": 0.9180000424385071,
+        "19": 0.9600000381469727
+      },
+      "llm_test_accuracy": {
+        "11": 0.9620000720024109,
+        "13": 0.9470000267028809,
+        "14": 0.9580000638961792,
+        "18": 0.9310000538825989,
+        "19": 0.9640000462532043
+      },
+      "llm_top_1_test_accuracy": {
+        "11": 0.558,
+        "13": 0.673,
+        "14": 0.656,
+        "18": 0.702,
+        "19": 0.793
+      },
+      "llm_top_2_test_accuracy": {
+        "11": 0.686,
+        "13": 0.713,
+        "14": 0.687,
+        "18": 0.724,
+        "19": 0.765
+      },
+      "llm_top_5_test_accuracy": {
+        "11": 0.782,
+        "13": 0.742,
+        "14": 0.716,
+        "18": 0.725,
+        "19": 0.831
+      },
+      "sae_top_1_test_accuracy": {
+        "11": 0.582,
+        "13": 0.684,
+        "14": 0.643,
+        "18": 0.698,
+        "19": 0.839
+      },
+      "sae_top_2_test_accuracy": {
+        "11": 0.745,
+        "13": 0.668,
+        "14": 0.868,
+        "18": 0.695,
+        "19": 0.841
+      },
+      "sae_top_5_test_accuracy": {
+        "11": 0.865,
+        "13": 0.694,
+        "14": 0.871,
+        "18": 0.729,
+        "19": 0.893
+      }
+    },
+    "LabHC/bias_in_bios_class_set3_results": {
+      "sae_test_accuracy": {
+        "20": 0.9570000171661377,
+        "21": 0.9200000166893005,
+        "22": 0.9100000262260437,
+        "25": 0.9570000171661377,
+        "26": 0.8850000500679016
+      },
+      "llm_test_accuracy": {
+        "20": 0.9610000252723694,
+        "21": 0.9140000343322754,
+        "22": 0.9170000553131104,
+        "25": 0.9630000591278076,
+        "26": 0.89000004529953
+      },
+      "llm_top_1_test_accuracy": {
+        "20": 0.711,
+        "21": 0.771,
+        "22": 0.637,
+        "25": 0.687,
+        "26": 0.626
+      },
+      "llm_top_2_test_accuracy": {
+        "20": 0.809,
+        "21": 0.764,
+        "22": 0.659,
+        "25": 0.766,
+        "26": 0.66
+      },
+      "llm_top_5_test_accuracy": {
+        "20": 0.858,
+        "21": 0.795,
+        "22": 0.715,
+        "25": 0.786,
+        "26": 0.679
+      },
+      "sae_top_1_test_accuracy": {
+        "20": 0.874,
+        "21": 0.532,
+        "22": 0.817,
+        "25": 0.876,
+        "26": 0.618
+      },
+      "sae_top_2_test_accuracy": {
+        "20": 0.884,
+        "21": 0.602,
+        "22": 0.862,
+        "25": 0.875,
+        "26": 0.703
+      },
+      "sae_top_5_test_accuracy": {
+        "20": 0.935,
+        "21": 0.847,
+        "22": 0.85,
+        "25": 0.887,
+        "26": 0.791
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_results": {
+      "sae_test_accuracy": {
+        "1": 0.9410000443458557,
+        "2": 0.9360000491142273,
+        "3": 0.9240000247955322,
+        "5": 0.9130000472068787,
+        "6": 0.8680000305175781
+      },
+      "llm_test_accuracy": {
+        "1": 0.9500000476837158,
+        "2": 0.937000036239624,
+        "3": 0.9260000586509705,
+        "5": 0.9120000600814819,
+        "6": 0.8560000658035278
+      },
+      "llm_top_1_test_accuracy": {
+        "1": 0.693,
+        "2": 0.607,
+        "3": 0.579,
+        "5": 0.577,
+        "6": 0.601
+      },
+      "llm_top_2_test_accuracy": {
+        "1": 0.747,
+        "2": 0.64,
+        "3": 0.607,
+        "5": 0.628,
+        "6": 0.619
+      },
+      "llm_top_5_test_accuracy": {
+        "1": 0.78,
+        "2": 0.657,
+        "3": 0.667,
+        "5": 0.659,
+        "6": 0.684
+      },
+      "sae_top_1_test_accuracy": {
+        "1": 0.872,
+        "2": 0.869,
+        "3": 0.561,
+        "5": 0.86,
+        "6": 0.593
+      },
+      "sae_top_2_test_accuracy": {
+        "1": 0.911,
+        "2": 0.873,
+        "3": 0.685,
+        "5": 0.869,
+        "6": 0.6
+      },
+      "sae_top_5_test_accuracy": {
+        "1": 0.924,
+        "2": 0.879,
+        "3": 0.767,
+        "5": 0.881,
+        "6": 0.747
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_sentiment_results": {
+      "sae_test_accuracy": {
+        "1.0": 0.971000075340271,
+        "5.0": 0.9720000624656677
+      },
+      "llm_test_accuracy": {
+        "1.0": 0.9820000529289246,
+        "5.0": 0.9820000529289246
+      },
+      "llm_top_1_test_accuracy": {
+        "1.0": 0.672,
+        "5.0": 0.672
+      },
+      "llm_top_2_test_accuracy": {
+        "1.0": 0.724,
+        "5.0": 0.724
+      },
+      "llm_top_5_test_accuracy": {
+        "1.0": 0.766,
+        "5.0": 0.766
+      },
+      "sae_top_1_test_accuracy": {
+        "1.0": 0.89,
+        "5.0": 0.89
+      },
+      "sae_top_2_test_accuracy": {
+        "1.0": 0.891,
+        "5.0": 0.891
+      },
+      "sae_top_5_test_accuracy": {
+        "1.0": 0.936,
+        "5.0": 0.936
+      }
+    },
+    "codeparrot/github-code_results": {
+      "sae_test_accuracy": {
+        "C": 0.956000030040741,
+        "Python": 0.9820000529289246,
+        "HTML": 0.9880000352859497,
+        "Java": 0.9570000171661377,
+        "PHP": 0.9620000720024109
+      },
+      "llm_test_accuracy": {
+        "C": 0.956000030040741,
+        "Python": 0.987000048160553,
+        "HTML": 0.9940000176429749,
+        "Java": 0.9610000252723694,
+        "PHP": 0.9590000510215759
+      },
+      "llm_top_1_test_accuracy": {
+        "C": 0.657,
+        "Python": 0.636,
+        "HTML": 0.733,
+        "Java": 0.616,
+        "PHP": 0.584
+      },
+      "llm_top_2_test_accuracy": {
+        "C": 0.671,
+        "Python": 0.668,
+        "HTML": 0.803,
+        "Java": 0.68,
+        "PHP": 0.642
+      },
+      "llm_top_5_test_accuracy": {
+        "C": 0.765,
+        "Python": 0.727,
+        "HTML": 0.943,
+        "Java": 0.735,
+        "PHP": 0.693
+      },
+      "sae_top_1_test_accuracy": {
+        "C": 0.624,
+        "Python": 0.629,
+        "HTML": 0.701,
+        "Java": 0.62,
+        "PHP": 0.601
+      },
+      "sae_top_2_test_accuracy": {
+        "C": 0.631,
+        "Python": 0.911,
+        "HTML": 0.734,
+        "Java": 0.653,
+        "PHP": 0.594
+      },
+      "sae_top_5_test_accuracy": {
+        "C": 0.684,
+        "Python": 0.936,
+        "HTML": 0.955,
+        "Java": 0.7,
+        "PHP": 0.921
+      }
+    },
+    "fancyzhx/ag_news_results": {
+      "sae_test_accuracy": {
+        "0": 0.9340000152587891,
+        "1": 0.984000027179718,
+        "2": 0.9320000410079956,
+        "3": 0.9480000734329224
+      },
+      "llm_test_accuracy": {
+        "0": 0.9390000700950623,
+        "1": 0.984000027179718,
+        "2": 0.9160000681877136,
+        "3": 0.9450000524520874
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.806,
+        "1": 0.662,
+        "2": 0.671,
+        "3": 0.791
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.796,
+        "1": 0.796,
+        "2": 0.694,
+        "3": 0.809
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.816,
+        "1": 0.885,
+        "2": 0.744,
+        "3": 0.84
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.743,
+        "1": 0.692,
+        "2": 0.671,
+        "3": 0.624
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.828,
+        "1": 0.682,
+        "2": 0.8,
+        "3": 0.64
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.842,
+        "1": 0.841,
+        "2": 0.836,
+        "3": 0.822
+      }
+    },
+    "Helsinki-NLP/europarl_results": {
+      "sae_test_accuracy": {
+        "en": 0.999000072479248,
+        "fr": 0.999000072479248,
+        "de": 0.9970000386238098,
+        "es": 0.999000072479248,
+        "nl": 1.0
+      },
+      "llm_test_accuracy": {
+        "en": 1.0,
+        "fr": 1.0,
+        "de": 1.0,
+        "es": 1.0,
+        "nl": 0.9980000257492065
+      },
+      "llm_top_1_test_accuracy": {
+        "en": 0.749,
+        "fr": 0.605,
+        "de": 0.741,
+        "es": 0.913,
+        "nl": 0.64
+      },
+      "llm_top_2_test_accuracy": {
+        "en": 0.831,
+        "fr": 0.607,
+        "de": 0.828,
+        "es": 0.915,
+        "nl": 0.753
+      },
+      "llm_top_5_test_accuracy": {
+        "en": 0.888,
+        "fr": 0.924,
+        "de": 0.882,
+        "es": 0.98,
+        "nl": 0.86
+      },
+      "sae_top_1_test_accuracy": {
+        "en": 0.84,
+        "fr": 0.992,
+        "de": 0.9,
+        "es": 0.877,
+        "nl": 0.746
+      },
+      "sae_top_2_test_accuracy": {
+        "en": 0.998,
+        "fr": 0.99,
+        "de": 0.907,
+        "es": 0.991,
+        "nl": 0.999
+      },
+      "sae_top_5_test_accuracy": {
+        "en": 1.0,
+        "fr": 0.994,
+        "de": 0.992,
+        "es": 0.996,
+        "nl": 0.999
+      }
+    }
+  }
+}

random_seed_eval_results/sparse_probing/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_3_custom_sae_eval_results.json

@@ -0,0 +1,670 @@
+{
+  "eval_type_id": "sparse_probing",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "LabHC/bias_in_bios_class_set2",
+      "LabHC/bias_in_bios_class_set3",
+      "canrager/amazon_reviews_mcauley_1and5",
+      "canrager/amazon_reviews_mcauley_1and5_sentiment",
+      "codeparrot/github-code",
+      "fancyzhx/ag_news",
+      "Helsinki-NLP/europarl"
+    ],
+    "probe_train_set_size": 4000,
+    "probe_test_set_size": 1000,
+    "context_length": 128,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "model_name": "gemma-2-2b",
+    "k_values": [
+      1,
+      2,
+      5
+    ],
+    "lower_vram_usage": false
+  },
+  "eval_id": "c9ab252b-9c9e-44fa-bcb2-af3a1b348f2b",
+  "datetime_epoch_millis": 1738794751340,
+  "eval_result_metrics": {
+    "llm": {
+      "llm_test_accuracy": 0.9582500416785479,
+      "llm_top_1_test_accuracy": 0.6746375,
+      "llm_top_2_test_accuracy": 0.7199437500000001,
+      "llm_top_5_test_accuracy": 0.78408125,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null
+    },
+    "sae": {
+      "sae_test_accuracy": 0.9557312924414874,
+      "sae_top_1_test_accuracy": 0.74515625,
+      "sae_top_2_test_accuracy": 0.8067187499999999,
+      "sae_top_5_test_accuracy": 0.8559499999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_results",
+      "llm_test_accuracy": 0.9694000363349915,
+      "llm_top_1_test_accuracy": 0.6436000000000001,
+      "llm_top_2_test_accuracy": 0.6874,
+      "llm_top_5_test_accuracy": 0.7908,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9648000359535217,
+      "sae_top_1_test_accuracy": 0.7744,
+      "sae_top_2_test_accuracy": 0.8164,
+      "sae_top_5_test_accuracy": 0.8907999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set2_results",
+      "llm_test_accuracy": 0.9524000525474549,
+      "llm_top_1_test_accuracy": 0.6764,
+      "llm_top_2_test_accuracy": 0.7150000000000001,
+      "llm_top_5_test_accuracy": 0.7592000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9498000383377075,
+      "sae_top_1_test_accuracy": 0.6796,
+      "sae_top_2_test_accuracy": 0.7634,
+      "sae_top_5_test_accuracy": 0.8123999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set3_results",
+      "llm_test_accuracy": 0.9290000438690186,
+      "llm_top_1_test_accuracy": 0.6864,
+      "llm_top_2_test_accuracy": 0.7316,
+      "llm_top_5_test_accuracy": 0.7666000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9282000422477722,
+      "sae_top_1_test_accuracy": 0.7568,
+      "sae_top_2_test_accuracy": 0.8108000000000001,
+      "sae_top_5_test_accuracy": 0.8694,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_results",
+      "llm_test_accuracy": 0.916200053691864,
+      "llm_top_1_test_accuracy": 0.6113999999999999,
+      "llm_top_2_test_accuracy": 0.6481999999999999,
+      "llm_top_5_test_accuracy": 0.6894,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9122000455856323,
+      "sae_top_1_test_accuracy": 0.7558,
+      "sae_top_2_test_accuracy": 0.7975999999999999,
+      "sae_top_5_test_accuracy": 0.8458,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_sentiment_results",
+      "llm_test_accuracy": 0.9820000529289246,
+      "llm_top_1_test_accuracy": 0.672,
+      "llm_top_2_test_accuracy": 0.724,
+      "llm_top_5_test_accuracy": 0.766,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9745000302791595,
+      "sae_top_1_test_accuracy": 0.819,
+      "sae_top_2_test_accuracy": 0.821,
+      "sae_top_5_test_accuracy": 0.833,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "codeparrot/github-code_results",
+      "llm_test_accuracy": 0.9714000344276428,
+      "llm_top_1_test_accuracy": 0.6452000000000001,
+      "llm_top_2_test_accuracy": 0.6928,
+      "llm_top_5_test_accuracy": 0.7726,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9682000517845154,
+      "sae_top_1_test_accuracy": 0.6362,
+      "sae_top_2_test_accuracy": 0.7532,
+      "sae_top_5_test_accuracy": 0.8176,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "fancyzhx/ag_news_results",
+      "llm_test_accuracy": 0.9460000544786453,
+      "llm_top_1_test_accuracy": 0.7325,
+      "llm_top_2_test_accuracy": 0.77375,
+      "llm_top_5_test_accuracy": 0.82125,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9497500509023666,
+      "sae_top_1_test_accuracy": 0.6652500000000001,
+      "sae_top_2_test_accuracy": 0.71375,
+      "sae_top_5_test_accuracy": 0.782,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "Helsinki-NLP/europarl_results",
+      "llm_test_accuracy": 0.9996000051498413,
+      "llm_top_1_test_accuracy": 0.7296,
+      "llm_top_2_test_accuracy": 0.7868,
+      "llm_top_5_test_accuracy": 0.9067999999999999,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9984000444412231,
+      "sae_top_1_test_accuracy": 0.8741999999999999,
+      "sae_top_2_test_accuracy": 0.9776,
+      "sae_top_5_test_accuracy": 0.9966000000000002,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_3",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1_results": {
+      "sae_test_accuracy": {
+        "0": 0.9470000267028809,
+        "1": 0.968000054359436,
+        "2": 0.9520000219345093,
+        "6": 0.9880000352859497,
+        "9": 0.9690000414848328
+      },
+      "llm_test_accuracy": {
+        "0": 0.9510000348091125,
+        "1": 0.9670000672340393,
+        "2": 0.9520000219345093,
+        "6": 0.9930000305175781,
+        "9": 0.984000027179718
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.568,
+        "1": 0.629,
+        "2": 0.679,
+        "6": 0.791,
+        "9": 0.551
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.585,
+        "1": 0.666,
+        "2": 0.673,
+        "6": 0.801,
+        "9": 0.712
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.72,
+        "1": 0.707,
+        "2": 0.764,
+        "6": 0.899,
+        "9": 0.864
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.599,
+        "1": 0.631,
+        "2": 0.893,
+        "6": 0.828,
+        "9": 0.921
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.678,
+        "1": 0.609,
+        "2": 0.891,
+        "6": 0.977,
+        "9": 0.927
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.842,
+        "1": 0.775,
+        "2": 0.91,
+        "6": 0.981,
+        "9": 0.946
+      }
+    },
+    "LabHC/bias_in_bios_class_set2_results": {
+      "sae_test_accuracy": {
+        "11": 0.9570000171661377,
+        "13": 0.9500000476837158,
+        "14": 0.9540000557899475,
+        "18": 0.9280000329017639,
+        "19": 0.9600000381469727
+      },
+      "llm_test_accuracy": {
+        "11": 0.9620000720024109,
+        "13": 0.9470000267028809,
+        "14": 0.9580000638961792,
+        "18": 0.9310000538825989,
+        "19": 0.9640000462532043
+      },
+      "llm_top_1_test_accuracy": {
+        "11": 0.558,
+        "13": 0.673,
+        "14": 0.656,
+        "18": 0.702,
+        "19": 0.793
+      },
+      "llm_top_2_test_accuracy": {
+        "11": 0.686,
+        "13": 0.713,
+        "14": 0.687,
+        "18": 0.724,
+        "19": 0.765
+      },
+      "llm_top_5_test_accuracy": {
+        "11": 0.782,
+        "13": 0.742,
+        "14": 0.716,
+        "18": 0.725,
+        "19": 0.831
+      },
+      "sae_top_1_test_accuracy": {
+        "11": 0.583,
+        "13": 0.686,
+        "14": 0.647,
+        "18": 0.679,
+        "19": 0.803
+      },
+      "sae_top_2_test_accuracy": {
+        "11": 0.741,
+        "13": 0.676,
+        "14": 0.868,
+        "18": 0.696,
+        "19": 0.836
+      },
+      "sae_top_5_test_accuracy": {
+        "11": 0.865,
+        "13": 0.688,
+        "14": 0.892,
+        "18": 0.732,
+        "19": 0.885
+      }
+    },
+    "LabHC/bias_in_bios_class_set3_results": {
+      "sae_test_accuracy": {
+        "20": 0.9510000348091125,
+        "21": 0.9260000586509705,
+        "22": 0.9150000214576721,
+        "25": 0.9600000381469727,
+        "26": 0.8890000581741333
+      },
+      "llm_test_accuracy": {
+        "20": 0.9610000252723694,
+        "21": 0.9140000343322754,
+        "22": 0.9170000553131104,
+        "25": 0.9630000591278076,
+        "26": 0.89000004529953
+      },
+      "llm_top_1_test_accuracy": {
+        "20": 0.711,
+        "21": 0.771,
+        "22": 0.637,
+        "25": 0.687,
+        "26": 0.626
+      },
+      "llm_top_2_test_accuracy": {
+        "20": 0.809,
+        "21": 0.764,
+        "22": 0.659,
+        "25": 0.766,
+        "26": 0.66
+      },
+      "llm_top_5_test_accuracy": {
+        "20": 0.858,
+        "21": 0.795,
+        "22": 0.715,
+        "25": 0.786,
+        "26": 0.679
+      },
+      "sae_top_1_test_accuracy": {
+        "20": 0.872,
+        "21": 0.521,
+        "22": 0.888,
+        "25": 0.875,
+        "26": 0.628
+      },
+      "sae_top_2_test_accuracy": {
+        "20": 0.917,
+        "21": 0.744,
+        "22": 0.894,
+        "25": 0.869,
+        "26": 0.63
+      },
+      "sae_top_5_test_accuracy": {
+        "20": 0.942,
+        "21": 0.846,
+        "22": 0.888,
+        "25": 0.889,
+        "26": 0.782
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_results": {
+      "sae_test_accuracy": {
+        "1": 0.9480000734329224,
+        "2": 0.9260000586509705,
+        "3": 0.9200000166893005,
+        "5": 0.909000039100647,
+        "6": 0.8580000400543213
+      },
+      "llm_test_accuracy": {
+        "1": 0.9500000476837158,
+        "2": 0.937000036239624,
+        "3": 0.9260000586509705,
+        "5": 0.9120000600814819,
+        "6": 0.8560000658035278
+      },
+      "llm_top_1_test_accuracy": {
+        "1": 0.693,
+        "2": 0.607,
+        "3": 0.579,
+        "5": 0.577,
+        "6": 0.601
+      },
+      "llm_top_2_test_accuracy": {
+        "1": 0.747,
+        "2": 0.64,
+        "3": 0.607,
+        "5": 0.628,
+        "6": 0.619
+      },
+      "llm_top_5_test_accuracy": {
+        "1": 0.78,
+        "2": 0.657,
+        "3": 0.667,
+        "5": 0.659,
+        "6": 0.684
+      },
+      "sae_top_1_test_accuracy": {
+        "1": 0.868,
+        "2": 0.846,
+        "3": 0.579,
+        "5": 0.889,
+        "6": 0.597
+      },
+      "sae_top_2_test_accuracy": {
+        "1": 0.899,
+        "2": 0.856,
+        "3": 0.674,
+        "5": 0.888,
+        "6": 0.671
+      },
+      "sae_top_5_test_accuracy": {
+        "1": 0.923,
+        "2": 0.882,
+        "3": 0.767,
+        "5": 0.896,
+        "6": 0.761
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_sentiment_results": {
+      "sae_test_accuracy": {
+        "1.0": 0.9750000238418579,
+        "5.0": 0.9740000367164612
+      },
+      "llm_test_accuracy": {
+        "1.0": 0.9820000529289246,
+        "5.0": 0.9820000529289246
+      },
+      "llm_top_1_test_accuracy": {
+        "1.0": 0.672,
+        "5.0": 0.672
+      },
+      "llm_top_2_test_accuracy": {
+        "1.0": 0.724,
+        "5.0": 0.724
+      },
+      "llm_top_5_test_accuracy": {
+        "1.0": 0.766,
+        "5.0": 0.766
+      },
+      "sae_top_1_test_accuracy": {
+        "1.0": 0.819,
+        "5.0": 0.819
+      },
+      "sae_top_2_test_accuracy": {
+        "1.0": 0.821,
+        "5.0": 0.821
+      },
+      "sae_top_5_test_accuracy": {
+        "1.0": 0.833,
+        "5.0": 0.833
+      }
+    },
+    "codeparrot/github-code_results": {
+      "sae_test_accuracy": {
+        "C": 0.9540000557899475,
+        "Python": 0.9810000658035278,
+        "HTML": 0.9880000352859497,
+        "Java": 0.9620000720024109,
+        "PHP": 0.956000030040741
+      },
+      "llm_test_accuracy": {
+        "C": 0.956000030040741,
+        "Python": 0.987000048160553,
+        "HTML": 0.9940000176429749,
+        "Java": 0.9610000252723694,
+        "PHP": 0.9590000510215759
+      },
+      "llm_top_1_test_accuracy": {
+        "C": 0.657,
+        "Python": 0.636,
+        "HTML": 0.733,
+        "Java": 0.616,
+        "PHP": 0.584
+      },
+      "llm_top_2_test_accuracy": {
+        "C": 0.671,
+        "Python": 0.668,
+        "HTML": 0.803,
+        "Java": 0.68,
+        "PHP": 0.642
+      },
+      "llm_top_5_test_accuracy": {
+        "C": 0.765,
+        "Python": 0.727,
+        "HTML": 0.943,
+        "Java": 0.735,
+        "PHP": 0.693
+      },
+      "sae_top_1_test_accuracy": {
+        "C": 0.615,
+        "Python": 0.631,
+        "HTML": 0.687,
+        "Java": 0.642,
+        "PHP": 0.606
+      },
+      "sae_top_2_test_accuracy": {
+        "C": 0.587,
+        "Python": 0.921,
+        "HTML": 0.692,
+        "Java": 0.653,
+        "PHP": 0.913
+      },
+      "sae_top_5_test_accuracy": {
+        "C": 0.681,
+        "Python": 0.935,
+        "HTML": 0.87,
+        "Java": 0.685,
+        "PHP": 0.917
+      }
+    },
+    "fancyzhx/ag_news_results": {
+      "sae_test_accuracy": {
+        "0": 0.9410000443458557,
+        "1": 0.9810000658035278,
+        "2": 0.9290000200271606,
+        "3": 0.9480000734329224
+      },
+      "llm_test_accuracy": {
+        "0": 0.9390000700950623,
+        "1": 0.984000027179718,
+        "2": 0.9160000681877136,
+        "3": 0.9450000524520874
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.806,
+        "1": 0.662,
+        "2": 0.671,
+        "3": 0.791
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.796,
+        "1": 0.796,
+        "2": 0.694,
+        "3": 0.809
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.816,
+        "1": 0.885,
+        "2": 0.744,
+        "3": 0.84
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.687,
+        "1": 0.7,
+        "2": 0.651,
+        "3": 0.623
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.839,
+        "1": 0.693,
+        "2": 0.695,
+        "3": 0.628
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.853,
+        "1": 0.808,
+        "2": 0.747,
+        "3": 0.72
+      }
+    },
+    "Helsinki-NLP/europarl_results": {
+      "sae_test_accuracy": {
+        "en": 0.9980000257492065,
+        "fr": 0.999000072479248,
+        "de": 0.9980000257492065,
+        "es": 0.999000072479248,
+        "nl": 0.9980000257492065
+      },
+      "llm_test_accuracy": {
+        "en": 1.0,
+        "fr": 1.0,
+        "de": 1.0,
+        "es": 1.0,
+        "nl": 0.9980000257492065
+      },
+      "llm_top_1_test_accuracy": {
+        "en": 0.749,
+        "fr": 0.605,
+        "de": 0.741,
+        "es": 0.913,
+        "nl": 0.64
+      },
+      "llm_top_2_test_accuracy": {
+        "en": 0.831,
+        "fr": 0.607,
+        "de": 0.828,
+        "es": 0.915,
+        "nl": 0.753
+      },
+      "llm_top_5_test_accuracy": {
+        "en": 0.888,
+        "fr": 0.924,
+        "de": 0.882,
+        "es": 0.98,
+        "nl": 0.86
+      },
+      "sae_top_1_test_accuracy": {
+        "en": 0.847,
+        "fr": 0.993,
+        "de": 0.908,
+        "es": 0.881,
+        "nl": 0.742
+      },
+      "sae_top_2_test_accuracy": {
+        "en": 0.999,
+        "fr": 0.995,
+        "de": 0.997,
+        "es": 0.899,
+        "nl": 0.998
+      },
+      "sae_top_5_test_accuracy": {
+        "en": 1.0,
+        "fr": 0.995,
+        "de": 0.996,
+        "es": 0.996,
+        "nl": 0.996
+      }
+    }
+  }
+}

random_seed_eval_results/sparse_probing/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_4_custom_sae_eval_results.json

@@ -0,0 +1,670 @@
+{
+  "eval_type_id": "sparse_probing",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "LabHC/bias_in_bios_class_set2",
+      "LabHC/bias_in_bios_class_set3",
+      "canrager/amazon_reviews_mcauley_1and5",
+      "canrager/amazon_reviews_mcauley_1and5_sentiment",
+      "codeparrot/github-code",
+      "fancyzhx/ag_news",
+      "Helsinki-NLP/europarl"
+    ],
+    "probe_train_set_size": 4000,
+    "probe_test_set_size": 1000,
+    "context_length": 128,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "model_name": "gemma-2-2b",
+    "k_values": [
+      1,
+      2,
+      5
+    ],
+    "lower_vram_usage": false
+  },
+  "eval_id": "a539ae76-2f7a-40a7-a02f-2a8ba952f201",
+  "datetime_epoch_millis": 1738794843835,
+  "eval_result_metrics": {
+    "llm": {
+      "llm_test_accuracy": 0.9582500416785479,
+      "llm_top_1_test_accuracy": 0.6746375,
+      "llm_top_2_test_accuracy": 0.7199437500000001,
+      "llm_top_5_test_accuracy": 0.78408125,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null
+    },
+    "sae": {
+      "sae_test_accuracy": 0.9551625456660985,
+      "sae_top_1_test_accuracy": 0.7446249999999999,
+      "sae_top_2_test_accuracy": 0.8137749999999999,
+      "sae_top_5_test_accuracy": 0.87466875,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_results",
+      "llm_test_accuracy": 0.9694000363349915,
+      "llm_top_1_test_accuracy": 0.6436000000000001,
+      "llm_top_2_test_accuracy": 0.6874,
+      "llm_top_5_test_accuracy": 0.7908,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.963200044631958,
+      "sae_top_1_test_accuracy": 0.7737999999999999,
+      "sae_top_2_test_accuracy": 0.8460000000000001,
+      "sae_top_5_test_accuracy": 0.9004,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set2_results",
+      "llm_test_accuracy": 0.9524000525474549,
+      "llm_top_1_test_accuracy": 0.6764,
+      "llm_top_2_test_accuracy": 0.7150000000000001,
+      "llm_top_5_test_accuracy": 0.7592000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9440000414848327,
+      "sae_top_1_test_accuracy": 0.6816000000000001,
+      "sae_top_2_test_accuracy": 0.79,
+      "sae_top_5_test_accuracy": 0.8244,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set3_results",
+      "llm_test_accuracy": 0.9290000438690186,
+      "llm_top_1_test_accuracy": 0.6864,
+      "llm_top_2_test_accuracy": 0.7316,
+      "llm_top_5_test_accuracy": 0.7666000000000001,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9278000473976136,
+      "sae_top_1_test_accuracy": 0.6816,
+      "sae_top_2_test_accuracy": 0.783,
+      "sae_top_5_test_accuracy": 0.8649999999999999,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_results",
+      "llm_test_accuracy": 0.916200053691864,
+      "llm_top_1_test_accuracy": 0.6113999999999999,
+      "llm_top_2_test_accuracy": 0.6481999999999999,
+      "llm_top_5_test_accuracy": 0.6894,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9196000576019288,
+      "sae_top_1_test_accuracy": 0.748,
+      "sae_top_2_test_accuracy": 0.7790000000000001,
+      "sae_top_5_test_accuracy": 0.8288,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_sentiment_results",
+      "llm_test_accuracy": 0.9820000529289246,
+      "llm_top_1_test_accuracy": 0.672,
+      "llm_top_2_test_accuracy": 0.724,
+      "llm_top_5_test_accuracy": 0.766,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9735000431537628,
+      "sae_top_1_test_accuracy": 0.882,
+      "sae_top_2_test_accuracy": 0.909,
+      "sae_top_5_test_accuracy": 0.947,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "codeparrot/github-code_results",
+      "llm_test_accuracy": 0.9714000344276428,
+      "llm_top_1_test_accuracy": 0.6452000000000001,
+      "llm_top_2_test_accuracy": 0.6928,
+      "llm_top_5_test_accuracy": 0.7726,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.966200053691864,
+      "sae_top_1_test_accuracy": 0.6402,
+      "sae_top_2_test_accuracy": 0.7772,
+      "sae_top_5_test_accuracy": 0.8308,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "fancyzhx/ag_news_results",
+      "llm_test_accuracy": 0.9460000544786453,
+      "llm_top_1_test_accuracy": 0.7325,
+      "llm_top_2_test_accuracy": 0.77375,
+      "llm_top_5_test_accuracy": 0.82125,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9480000287294388,
+      "sae_top_1_test_accuracy": 0.693,
+      "sae_top_2_test_accuracy": 0.696,
+      "sae_top_5_test_accuracy": 0.81675,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    },
+    {
+      "dataset_name": "Helsinki-NLP/europarl_results",
+      "llm_test_accuracy": 0.9996000051498413,
+      "llm_top_1_test_accuracy": 0.7296,
+      "llm_top_2_test_accuracy": 0.7868,
+      "llm_top_5_test_accuracy": 0.9067999999999999,
+      "llm_top_10_test_accuracy": null,
+      "llm_top_20_test_accuracy": null,
+      "llm_top_50_test_accuracy": null,
+      "llm_top_100_test_accuracy": null,
+      "sae_test_accuracy": 0.9990000486373901,
+      "sae_top_1_test_accuracy": 0.8568,
+      "sae_top_2_test_accuracy": 0.9299999999999999,
+      "sae_top_5_test_accuracy": 0.9842000000000001,
+      "sae_top_10_test_accuracy": null,
+      "sae_top_20_test_accuracy": null,
+      "sae_top_50_test_accuracy": null,
+      "sae_top_100_test_accuracy": null
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_4",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1_results": {
+      "sae_test_accuracy": {
+        "0": 0.9430000185966492,
+        "1": 0.9600000381469727,
+        "2": 0.9480000734329224,
+        "6": 0.9880000352859497,
+        "9": 0.9770000576972961
+      },
+      "llm_test_accuracy": {
+        "0": 0.9510000348091125,
+        "1": 0.9670000672340393,
+        "2": 0.9520000219345093,
+        "6": 0.9930000305175781,
+        "9": 0.984000027179718
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.568,
+        "1": 0.629,
+        "2": 0.679,
+        "6": 0.791,
+        "9": 0.551
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.585,
+        "1": 0.666,
+        "2": 0.673,
+        "6": 0.801,
+        "9": 0.712
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.72,
+        "1": 0.707,
+        "2": 0.764,
+        "6": 0.899,
+        "9": 0.864
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.596,
+        "1": 0.637,
+        "2": 0.871,
+        "6": 0.832,
+        "9": 0.933
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.645,
+        "1": 0.798,
+        "2": 0.882,
+        "6": 0.98,
+        "9": 0.925
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.843,
+        "1": 0.841,
+        "2": 0.893,
+        "6": 0.981,
+        "9": 0.944
+      }
+    },
+    "LabHC/bias_in_bios_class_set2_results": {
+      "sae_test_accuracy": {
+        "11": 0.9590000510215759,
+        "13": 0.9450000524520874,
+        "14": 0.9540000557899475,
+        "18": 0.9100000262260437,
+        "19": 0.9520000219345093
+      },
+      "llm_test_accuracy": {
+        "11": 0.9620000720024109,
+        "13": 0.9470000267028809,
+        "14": 0.9580000638961792,
+        "18": 0.9310000538825989,
+        "19": 0.9640000462532043
+      },
+      "llm_top_1_test_accuracy": {
+        "11": 0.558,
+        "13": 0.673,
+        "14": 0.656,
+        "18": 0.702,
+        "19": 0.793
+      },
+      "llm_top_2_test_accuracy": {
+        "11": 0.686,
+        "13": 0.713,
+        "14": 0.687,
+        "18": 0.724,
+        "19": 0.765
+      },
+      "llm_top_5_test_accuracy": {
+        "11": 0.782,
+        "13": 0.742,
+        "14": 0.716,
+        "18": 0.725,
+        "19": 0.831
+      },
+      "sae_top_1_test_accuracy": {
+        "11": 0.59,
+        "13": 0.684,
+        "14": 0.636,
+        "18": 0.702,
+        "19": 0.796
+      },
+      "sae_top_2_test_accuracy": {
+        "11": 0.855,
+        "13": 0.678,
+        "14": 0.891,
+        "18": 0.685,
+        "19": 0.841
+      },
+      "sae_top_5_test_accuracy": {
+        "11": 0.944,
+        "13": 0.688,
+        "14": 0.901,
+        "18": 0.744,
+        "19": 0.845
+      }
+    },
+    "LabHC/bias_in_bios_class_set3_results": {
+      "sae_test_accuracy": {
+        "20": 0.9570000171661377,
+        "21": 0.921000063419342,
+        "22": 0.9140000343322754,
+        "25": 0.9580000638961792,
+        "26": 0.8890000581741333
+      },
+      "llm_test_accuracy": {
+        "20": 0.9610000252723694,
+        "21": 0.9140000343322754,
+        "22": 0.9170000553131104,
+        "25": 0.9630000591278076,
+        "26": 0.89000004529953
+      },
+      "llm_top_1_test_accuracy": {
+        "20": 0.711,
+        "21": 0.771,
+        "22": 0.637,
+        "25": 0.687,
+        "26": 0.626
+      },
+      "llm_top_2_test_accuracy": {
+        "20": 0.809,
+        "21": 0.764,
+        "22": 0.659,
+        "25": 0.766,
+        "26": 0.66
+      },
+      "llm_top_5_test_accuracy": {
+        "20": 0.858,
+        "21": 0.795,
+        "22": 0.715,
+        "25": 0.786,
+        "26": 0.679
+      },
+      "sae_top_1_test_accuracy": {
+        "20": 0.874,
+        "21": 0.514,
+        "22": 0.681,
+        "25": 0.713,
+        "26": 0.626
+      },
+      "sae_top_2_test_accuracy": {
+        "20": 0.881,
+        "21": 0.829,
+        "22": 0.703,
+        "25": 0.871,
+        "26": 0.631
+      },
+      "sae_top_5_test_accuracy": {
+        "20": 0.942,
+        "21": 0.842,
+        "22": 0.872,
+        "25": 0.885,
+        "26": 0.784
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_results": {
+      "sae_test_accuracy": {
+        "1": 0.9480000734329224,
+        "2": 0.9460000395774841,
+        "3": 0.9170000553131104,
+        "5": 0.9080000519752502,
+        "6": 0.8790000677108765
+      },
+      "llm_test_accuracy": {
+        "1": 0.9500000476837158,
+        "2": 0.937000036239624,
+        "3": 0.9260000586509705,
+        "5": 0.9120000600814819,
+        "6": 0.8560000658035278
+      },
+      "llm_top_1_test_accuracy": {
+        "1": 0.693,
+        "2": 0.607,
+        "3": 0.579,
+        "5": 0.577,
+        "6": 0.601
+      },
+      "llm_top_2_test_accuracy": {
+        "1": 0.747,
+        "2": 0.64,
+        "3": 0.607,
+        "5": 0.628,
+        "6": 0.619
+      },
+      "llm_top_5_test_accuracy": {
+        "1": 0.78,
+        "2": 0.657,
+        "3": 0.667,
+        "5": 0.659,
+        "6": 0.684
+      },
+      "sae_top_1_test_accuracy": {
+        "1": 0.859,
+        "2": 0.858,
+        "3": 0.578,
+        "5": 0.862,
+        "6": 0.583
+      },
+      "sae_top_2_test_accuracy": {
+        "1": 0.888,
+        "2": 0.869,
+        "3": 0.589,
+        "5": 0.857,
+        "6": 0.692
+      },
+      "sae_top_5_test_accuracy": {
+        "1": 0.925,
+        "2": 0.865,
+        "3": 0.739,
+        "5": 0.866,
+        "6": 0.749
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5_sentiment_results": {
+      "sae_test_accuracy": {
+        "1.0": 0.9740000367164612,
+        "5.0": 0.9730000495910645
+      },
+      "llm_test_accuracy": {
+        "1.0": 0.9820000529289246,
+        "5.0": 0.9820000529289246
+      },
+      "llm_top_1_test_accuracy": {
+        "1.0": 0.672,
+        "5.0": 0.672
+      },
+      "llm_top_2_test_accuracy": {
+        "1.0": 0.724,
+        "5.0": 0.724
+      },
+      "llm_top_5_test_accuracy": {
+        "1.0": 0.766,
+        "5.0": 0.766
+      },
+      "sae_top_1_test_accuracy": {
+        "1.0": 0.882,
+        "5.0": 0.882
+      },
+      "sae_top_2_test_accuracy": {
+        "1.0": 0.909,
+        "5.0": 0.909
+      },
+      "sae_top_5_test_accuracy": {
+        "1.0": 0.947,
+        "5.0": 0.947
+      }
+    },
+    "codeparrot/github-code_results": {
+      "sae_test_accuracy": {
+        "C": 0.9450000524520874,
+        "Python": 0.9820000529289246,
+        "HTML": 0.984000027179718,
+        "Java": 0.9620000720024109,
+        "PHP": 0.9580000638961792
+      },
+      "llm_test_accuracy": {
+        "C": 0.956000030040741,
+        "Python": 0.987000048160553,
+        "HTML": 0.9940000176429749,
+        "Java": 0.9610000252723694,
+        "PHP": 0.9590000510215759
+      },
+      "llm_top_1_test_accuracy": {
+        "C": 0.657,
+        "Python": 0.636,
+        "HTML": 0.733,
+        "Java": 0.616,
+        "PHP": 0.584
+      },
+      "llm_top_2_test_accuracy": {
+        "C": 0.671,
+        "Python": 0.668,
+        "HTML": 0.803,
+        "Java": 0.68,
+        "PHP": 0.642
+      },
+      "llm_top_5_test_accuracy": {
+        "C": 0.765,
+        "Python": 0.727,
+        "HTML": 0.943,
+        "Java": 0.735,
+        "PHP": 0.693
+      },
+      "sae_top_1_test_accuracy": {
+        "C": 0.631,
+        "Python": 0.643,
+        "HTML": 0.687,
+        "Java": 0.64,
+        "PHP": 0.6
+      },
+      "sae_top_2_test_accuracy": {
+        "C": 0.647,
+        "Python": 0.921,
+        "HTML": 0.737,
+        "Java": 0.656,
+        "PHP": 0.925
+      },
+      "sae_top_5_test_accuracy": {
+        "C": 0.697,
+        "Python": 0.934,
+        "HTML": 0.905,
+        "Java": 0.69,
+        "PHP": 0.928
+      }
+    },
+    "fancyzhx/ag_news_results": {
+      "sae_test_accuracy": {
+        "0": 0.9280000329017639,
+        "1": 0.984000027179718,
+        "2": 0.9330000281333923,
+        "3": 0.9470000267028809
+      },
+      "llm_test_accuracy": {
+        "0": 0.9390000700950623,
+        "1": 0.984000027179718,
+        "2": 0.9160000681877136,
+        "3": 0.9450000524520874
+      },
+      "llm_top_1_test_accuracy": {
+        "0": 0.806,
+        "1": 0.662,
+        "2": 0.671,
+        "3": 0.791
+      },
+      "llm_top_2_test_accuracy": {
+        "0": 0.796,
+        "1": 0.796,
+        "2": 0.694,
+        "3": 0.809
+      },
+      "llm_top_5_test_accuracy": {
+        "0": 0.816,
+        "1": 0.885,
+        "2": 0.744,
+        "3": 0.84
+      },
+      "sae_top_1_test_accuracy": {
+        "0": 0.781,
+        "1": 0.691,
+        "2": 0.67,
+        "3": 0.63
+      },
+      "sae_top_2_test_accuracy": {
+        "0": 0.762,
+        "1": 0.674,
+        "2": 0.698,
+        "3": 0.65
+      },
+      "sae_top_5_test_accuracy": {
+        "0": 0.84,
+        "1": 0.904,
+        "2": 0.796,
+        "3": 0.727
+      }
+    },
+    "Helsinki-NLP/europarl_results": {
+      "sae_test_accuracy": {
+        "en": 0.999000072479248,
+        "fr": 0.999000072479248,
+        "de": 1.0,
+        "es": 0.9980000257492065,
+        "nl": 0.999000072479248
+      },
+      "llm_test_accuracy": {
+        "en": 1.0,
+        "fr": 1.0,
+        "de": 1.0,
+        "es": 1.0,
+        "nl": 0.9980000257492065
+      },
+      "llm_top_1_test_accuracy": {
+        "en": 0.749,
+        "fr": 0.605,
+        "de": 0.741,
+        "es": 0.913,
+        "nl": 0.64
+      },
+      "llm_top_2_test_accuracy": {
+        "en": 0.831,
+        "fr": 0.607,
+        "de": 0.828,
+        "es": 0.915,
+        "nl": 0.753
+      },
+      "llm_top_5_test_accuracy": {
+        "en": 0.888,
+        "fr": 0.924,
+        "de": 0.882,
+        "es": 0.98,
+        "nl": 0.86
+      },
+      "sae_top_1_test_accuracy": {
+        "en": 0.852,
+        "fr": 0.994,
+        "de": 0.9,
+        "es": 0.802,
+        "nl": 0.736
+      },
+      "sae_top_2_test_accuracy": {
+        "en": 0.838,
+        "fr": 0.994,
+        "de": 0.92,
+        "es": 0.9,
+        "nl": 0.998
+      },
+      "sae_top_5_test_accuracy": {
+        "en": 0.997,
+        "fr": 0.994,
+        "de": 0.938,
+        "es": 0.995,
+        "nl": 0.997
+      }
+    }
+  }
+}

random_seed_eval_results/tpp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_0_custom_sae_eval_results.json

@@ -0,0 +1,414 @@
+{
+  "eval_type_id": "tpp",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": false,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "945968cc-e6d4-4ea1-9ea9-30e085bc5389",
+  "datetime_epoch_millis": 1738793323541,
+  "eval_result_metrics": {
+    "tpp_metrics": {
+      "tpp_threshold_2_total_metric": 0.008049994707107544,
+      "tpp_threshold_2_intended_diff_only": 0.010899996757507325,
+      "tpp_threshold_2_unintended_diff_only": 0.0028500020503997806,
+      "tpp_threshold_5_total_metric": 0.011550003290176391,
+      "tpp_threshold_5_intended_diff_only": 0.01510000228881836,
+      "tpp_threshold_5_unintended_diff_only": 0.0035499989986419677,
+      "tpp_threshold_10_total_metric": 0.02980000078678131,
+      "tpp_threshold_10_intended_diff_only": 0.03420000672340393,
+      "tpp_threshold_10_unintended_diff_only": 0.00440000593662262,
+      "tpp_threshold_20_total_metric": 0.058100007474422455,
+      "tpp_threshold_20_intended_diff_only": 0.06480000615119935,
+      "tpp_threshold_20_unintended_diff_only": 0.006699998676776887,
+      "tpp_threshold_50_total_metric": 0.14129999876022337,
+      "tpp_threshold_50_intended_diff_only": 0.15059999823570253,
+      "tpp_threshold_50_unintended_diff_only": 0.009299999475479126,
+      "tpp_threshold_100_total_metric": 0.2181250184774399,
+      "tpp_threshold_100_intended_diff_only": 0.2313000202178955,
+      "tpp_threshold_100_unintended_diff_only": 0.013175001740455626,
+      "tpp_threshold_500_total_metric": 0.3984750136733055,
+      "tpp_threshold_500_intended_diff_only": 0.41890001893043516,
+      "tpp_threshold_500_unintended_diff_only": 0.020425005257129668
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
+      "tpp_threshold_2_total_metric": 0.007599985599517823,
+      "tpp_threshold_2_intended_diff_only": 0.009999990463256836,
+      "tpp_threshold_2_unintended_diff_only": 0.0024000048637390138,
+      "tpp_threshold_5_total_metric": 0.013449999690055846,
+      "tpp_threshold_5_intended_diff_only": 0.016600000858306884,
+      "tpp_threshold_5_unintended_diff_only": 0.0031500011682510376,
+      "tpp_threshold_10_total_metric": 0.028599995374679565,
+      "tpp_threshold_10_intended_diff_only": 0.031599998474121094,
+      "tpp_threshold_10_unintended_diff_only": 0.003000003099441528,
+      "tpp_threshold_20_total_metric": 0.07145001292228699,
+      "tpp_threshold_20_intended_diff_only": 0.07780001163482667,
+      "tpp_threshold_20_unintended_diff_only": 0.006349998712539673,
+      "tpp_threshold_50_total_metric": 0.15539998710155487,
+      "tpp_threshold_50_intended_diff_only": 0.16179999113082885,
+      "tpp_threshold_50_unintended_diff_only": 0.006400004029273987,
+      "tpp_threshold_100_total_metric": 0.2591500222682953,
+      "tpp_threshold_100_intended_diff_only": 0.2678000211715698,
+      "tpp_threshold_100_unintended_diff_only": 0.008649998903274536,
+      "tpp_threshold_500_total_metric": 0.4453500181436539,
+      "tpp_threshold_500_intended_diff_only": 0.4556000232696533,
+      "tpp_threshold_500_unintended_diff_only": 0.01025000512599945
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
+      "tpp_threshold_2_total_metric": 0.008500003814697265,
+      "tpp_threshold_2_intended_diff_only": 0.011800003051757813,
+      "tpp_threshold_2_unintended_diff_only": 0.003299999237060547,
+      "tpp_threshold_5_total_metric": 0.009650006890296936,
+      "tpp_threshold_5_intended_diff_only": 0.013600003719329835,
+      "tpp_threshold_5_unintended_diff_only": 0.003949996829032898,
+      "tpp_threshold_10_total_metric": 0.031000006198883056,
+      "tpp_threshold_10_intended_diff_only": 0.03680001497268677,
+      "tpp_threshold_10_unintended_diff_only": 0.005800008773803711,
+      "tpp_threshold_20_total_metric": 0.04475000202655792,
+      "tpp_threshold_20_intended_diff_only": 0.051800000667572024,
+      "tpp_threshold_20_unintended_diff_only": 0.0070499986410140995,
+      "tpp_threshold_50_total_metric": 0.1272000104188919,
+      "tpp_threshold_50_intended_diff_only": 0.13940000534057617,
+      "tpp_threshold_50_unintended_diff_only": 0.012199994921684266,
+      "tpp_threshold_100_total_metric": 0.17710001468658448,
+      "tpp_threshold_100_intended_diff_only": 0.1948000192642212,
+      "tpp_threshold_100_unintended_diff_only": 0.017700004577636718,
+      "tpp_threshold_500_total_metric": 0.35160000920295714,
+      "tpp_threshold_500_intended_diff_only": 0.38220001459121705,
+      "tpp_threshold_500_unintended_diff_only": 0.03060000538825989
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_0",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1": {
+      "0": {
+        "tpp_threshold_2_total_metric": 0.01075001060962677,
+        "tpp_threshold_2_intended_diff_only": 0.013000011444091797,
+        "tpp_threshold_2_unintended_diff_only": 0.002250000834465027,
+        "tpp_threshold_5_total_metric": 0.014250010251998901,
+        "tpp_threshold_5_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_5_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_10_total_metric": 0.011250033974647522,
+        "tpp_threshold_10_intended_diff_only": 0.01500004529953003,
+        "tpp_threshold_10_unintended_diff_only": 0.0037500113248825073,
+        "tpp_threshold_20_total_metric": 0.02625003457069397,
+        "tpp_threshold_20_intended_diff_only": 0.029000043869018555,
+        "tpp_threshold_20_unintended_diff_only": 0.002750009298324585,
+        "tpp_threshold_50_total_metric": 0.05324999988079071,
+        "tpp_threshold_50_intended_diff_only": 0.0559999942779541,
+        "tpp_threshold_50_unintended_diff_only": 0.002749994397163391,
+        "tpp_threshold_100_total_metric": 0.12700004875659943,
+        "tpp_threshold_100_intended_diff_only": 0.13100004196166992,
+        "tpp_threshold_100_unintended_diff_only": 0.003999993205070496,
+        "tpp_threshold_500_total_metric": 0.4242500364780426,
+        "tpp_threshold_500_intended_diff_only": 0.4280000329017639,
+        "tpp_threshold_500_unintended_diff_only": 0.0037499964237213135
+      },
+      "1": {
+        "tpp_threshold_2_total_metric": 0.0059999823570251465,
+        "tpp_threshold_2_intended_diff_only": 0.0059999823570251465,
+        "tpp_threshold_2_unintended_diff_only": 0.0,
+        "tpp_threshold_5_total_metric": 0.0052499473094940186,
+        "tpp_threshold_5_intended_diff_only": 0.006999969482421875,
+        "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564,
+        "tpp_threshold_10_total_metric": 0.013749957084655762,
+        "tpp_threshold_10_intended_diff_only": 0.012999951839447021,
+        "tpp_threshold_10_unintended_diff_only": -0.0007500052452087402,
+        "tpp_threshold_20_total_metric": 0.05449996888637543,
+        "tpp_threshold_20_intended_diff_only": 0.06199997663497925,
+        "tpp_threshold_20_unintended_diff_only": 0.007500007748603821,
+        "tpp_threshold_50_total_metric": 0.143249973654747,
+        "tpp_threshold_50_intended_diff_only": 0.1499999761581421,
+        "tpp_threshold_50_unintended_diff_only": 0.006750002503395081,
+        "tpp_threshold_100_total_metric": 0.22350002825260162,
+        "tpp_threshold_100_intended_diff_only": 0.23000001907348633,
+        "tpp_threshold_100_unintended_diff_only": 0.006499990820884705,
+        "tpp_threshold_500_total_metric": 0.4404999762773514,
+        "tpp_threshold_500_intended_diff_only": 0.44999998807907104,
+        "tpp_threshold_500_unintended_diff_only": 0.009500011801719666
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.006999969482421875,
+        "tpp_threshold_2_intended_diff_only": 0.010999977588653564,
+        "tpp_threshold_2_unintended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_5_total_metric": 0.021250009536743164,
+        "tpp_threshold_5_intended_diff_only": 0.027000010013580322,
+        "tpp_threshold_5_unintended_diff_only": 0.005750000476837158,
+        "tpp_threshold_10_total_metric": 0.04700000584125519,
+        "tpp_threshold_10_intended_diff_only": 0.050000011920928955,
+        "tpp_threshold_10_unintended_diff_only": 0.003000006079673767,
+        "tpp_threshold_20_total_metric": 0.06775002181529999,
+        "tpp_threshold_20_intended_diff_only": 0.0690000057220459,
+        "tpp_threshold_20_unintended_diff_only": 0.0012499839067459106,
+        "tpp_threshold_50_total_metric": 0.127749964594841,
+        "tpp_threshold_50_intended_diff_only": 0.12699997425079346,
+        "tpp_threshold_50_unintended_diff_only": -0.0007499903440475464,
+        "tpp_threshold_100_total_metric": 0.2237500101327896,
+        "tpp_threshold_100_intended_diff_only": 0.22699999809265137,
+        "tpp_threshold_100_unintended_diff_only": 0.0032499879598617554,
+        "tpp_threshold_500_total_metric": 0.43700000643730164,
+        "tpp_threshold_500_intended_diff_only": 0.4440000057220459,
+        "tpp_threshold_500_unintended_diff_only": 0.006999999284744263
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.002749994397163391,
+        "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_2_unintended_diff_only": 0.0012500137090682983,
+        "tpp_threshold_5_total_metric": 0.0030000507831573486,
+        "tpp_threshold_5_intended_diff_only": 0.0020000338554382324,
+        "tpp_threshold_5_unintended_diff_only": -0.0010000169277191162,
+        "tpp_threshold_10_total_metric": 0.0065000057220458984,
+        "tpp_threshold_10_intended_diff_only": 0.009000003337860107,
+        "tpp_threshold_10_unintended_diff_only": 0.002499997615814209,
+        "tpp_threshold_20_total_metric": 0.0975000262260437,
+        "tpp_threshold_20_intended_diff_only": 0.11000001430511475,
+        "tpp_threshold_20_unintended_diff_only": 0.012499988079071045,
+        "tpp_threshold_50_total_metric": 0.23750002682209015,
+        "tpp_threshold_50_intended_diff_only": 0.25200003385543823,
+        "tpp_threshold_50_unintended_diff_only": 0.014500007033348083,
+        "tpp_threshold_100_total_metric": 0.34950003027915955,
+        "tpp_threshold_100_intended_diff_only": 0.3670000433921814,
+        "tpp_threshold_100_unintended_diff_only": 0.01750001311302185,
+        "tpp_threshold_500_total_metric": 0.46025004982948303,
+        "tpp_threshold_500_intended_diff_only": 0.47700005769729614,
+        "tpp_threshold_500_unintended_diff_only": 0.01675000786781311
+      },
+      "9": {
+        "tpp_threshold_2_total_metric": 0.011499971151351929,
+        "tpp_threshold_2_intended_diff_only": 0.015999972820281982,
+        "tpp_threshold_2_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_5_total_metric": 0.023499980568885803,
+        "tpp_threshold_5_intended_diff_only": 0.02899998426437378,
+        "tpp_threshold_5_unintended_diff_only": 0.005500003695487976,
+        "tpp_threshold_10_total_metric": 0.06449997425079346,
+        "tpp_threshold_10_intended_diff_only": 0.07099997997283936,
+        "tpp_threshold_10_unintended_diff_only": 0.0065000057220458984,
+        "tpp_threshold_20_total_metric": 0.11125001311302185,
+        "tpp_threshold_20_intended_diff_only": 0.11900001764297485,
+        "tpp_threshold_20_unintended_diff_only": 0.007750004529953003,
+        "tpp_threshold_50_total_metric": 0.21524997055530548,
+        "tpp_threshold_50_intended_diff_only": 0.2239999771118164,
+        "tpp_threshold_50_unintended_diff_only": 0.008750006556510925,
+        "tpp_threshold_100_total_metric": 0.37199999392032623,
+        "tpp_threshold_100_intended_diff_only": 0.3840000033378601,
+        "tpp_threshold_100_unintended_diff_only": 0.012000009417533875,
+        "tpp_threshold_500_total_metric": 0.4647500216960907,
+        "tpp_threshold_500_intended_diff_only": 0.4790000319480896,
+        "tpp_threshold_500_unintended_diff_only": 0.014250010251998901
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5": {
+      "1": {
+        "tpp_threshold_2_total_metric": 0.006000041961669922,
+        "tpp_threshold_2_intended_diff_only": 0.010000050067901611,
+        "tpp_threshold_2_unintended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_5_total_metric": 0.010250017046928406,
+        "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
+        "tpp_threshold_5_unintended_diff_only": 0.002749994397163391,
+        "tpp_threshold_10_total_metric": 0.011750057339668274,
+        "tpp_threshold_10_intended_diff_only": 0.01900005340576172,
+        "tpp_threshold_10_unintended_diff_only": 0.007249996066093445,
+        "tpp_threshold_20_total_metric": 0.026500031352043152,
+        "tpp_threshold_20_intended_diff_only": 0.03100001811981201,
+        "tpp_threshold_20_unintended_diff_only": 0.00449998676776886,
+        "tpp_threshold_50_total_metric": 0.0532500296831131,
+        "tpp_threshold_50_intended_diff_only": 0.057000041007995605,
+        "tpp_threshold_50_unintended_diff_only": 0.0037500113248825073,
+        "tpp_threshold_100_total_metric": 0.09050005674362183,
+        "tpp_threshold_100_intended_diff_only": 0.10200005769729614,
+        "tpp_threshold_100_unintended_diff_only": 0.011500000953674316,
+        "tpp_threshold_500_total_metric": 0.37300005555152893,
+        "tpp_threshold_500_intended_diff_only": 0.38600003719329834,
+        "tpp_threshold_500_unintended_diff_only": 0.01299998164176941
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.006749987602233887,
+        "tpp_threshold_2_intended_diff_only": 0.006999969482421875,
+        "tpp_threshold_2_unintended_diff_only": 0.0002499818801879883,
+        "tpp_threshold_5_total_metric": -0.0059999823570251465,
+        "tpp_threshold_5_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_5_unintended_diff_only": 0.009999990463256836,
+        "tpp_threshold_10_total_metric": 0.029999956488609314,
+        "tpp_threshold_10_intended_diff_only": 0.0339999794960022,
+        "tpp_threshold_10_unintended_diff_only": 0.004000023007392883,
+        "tpp_threshold_20_total_metric": 0.04174995422363281,
+        "tpp_threshold_20_intended_diff_only": 0.04499995708465576,
+        "tpp_threshold_20_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_50_total_metric": 0.09049999713897705,
+        "tpp_threshold_50_intended_diff_only": 0.10699999332427979,
+        "tpp_threshold_50_unintended_diff_only": 0.016499996185302734,
+        "tpp_threshold_100_total_metric": 0.14699998497962952,
+        "tpp_threshold_100_intended_diff_only": 0.171999990940094,
+        "tpp_threshold_100_unintended_diff_only": 0.025000005960464478,
+        "tpp_threshold_500_total_metric": 0.3572499603033066,
+        "tpp_threshold_500_intended_diff_only": 0.390999972820282,
+        "tpp_threshold_500_unintended_diff_only": 0.0337500125169754
+      },
+      "3": {
+        "tpp_threshold_2_total_metric": -0.00849999487400055,
+        "tpp_threshold_2_intended_diff_only": -0.004999995231628418,
+        "tpp_threshold_2_unintended_diff_only": 0.0034999996423721313,
+        "tpp_threshold_5_total_metric": -0.0012499839067459106,
+        "tpp_threshold_5_intended_diff_only": -0.0009999871253967285,
+        "tpp_threshold_5_unintended_diff_only": 0.00024999678134918213,
+        "tpp_threshold_10_total_metric": 0.01599995791912079,
+        "tpp_threshold_10_intended_diff_only": 0.0209999680519104,
+        "tpp_threshold_10_unintended_diff_only": 0.005000010132789612,
+        "tpp_threshold_20_total_metric": 0.0052499920129776,
+        "tpp_threshold_20_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_20_unintended_diff_only": 0.008750006556510925,
+        "tpp_threshold_50_total_metric": 0.07024997472763062,
+        "tpp_threshold_50_intended_diff_only": 0.08099997043609619,
+        "tpp_threshold_50_unintended_diff_only": 0.010749995708465576,
+        "tpp_threshold_100_total_metric": 0.1054999977350235,
+        "tpp_threshold_100_intended_diff_only": 0.12099999189376831,
+        "tpp_threshold_100_unintended_diff_only": 0.015499994158744812,
+        "tpp_threshold_500_total_metric": 0.33024996519088745,
+        "tpp_threshold_500_intended_diff_only": 0.3619999885559082,
+        "tpp_threshold_500_unintended_diff_only": 0.03175002336502075
+      },
+      "5": {
+        "tpp_threshold_2_total_metric": 0.019249990582466125,
+        "tpp_threshold_2_intended_diff_only": 0.02399998903274536,
+        "tpp_threshold_2_unintended_diff_only": 0.004749998450279236,
+        "tpp_threshold_5_total_metric": 0.026249989867210388,
+        "tpp_threshold_5_intended_diff_only": 0.03299999237060547,
+        "tpp_threshold_5_unintended_diff_only": 0.006750002503395081,
+        "tpp_threshold_10_total_metric": 0.03400002419948578,
+        "tpp_threshold_10_intended_diff_only": 0.04300004243850708,
+        "tpp_threshold_10_unintended_diff_only": 0.009000018239021301,
+        "tpp_threshold_20_total_metric": 0.054000020027160645,
+        "tpp_threshold_20_intended_diff_only": 0.06300002336502075,
+        "tpp_threshold_20_unintended_diff_only": 0.009000003337860107,
+        "tpp_threshold_50_total_metric": 0.16875000298023224,
+        "tpp_threshold_50_intended_diff_only": 0.1809999942779541,
+        "tpp_threshold_50_unintended_diff_only": 0.012249991297721863,
+        "tpp_threshold_100_total_metric": 0.2290000468492508,
+        "tpp_threshold_100_intended_diff_only": 0.24600005149841309,
+        "tpp_threshold_100_unintended_diff_only": 0.017000004649162292,
+        "tpp_threshold_500_total_metric": 0.35500001907348633,
+        "tpp_threshold_500_intended_diff_only": 0.3960000276565552,
+        "tpp_threshold_500_unintended_diff_only": 0.04100000858306885
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.018999993801116943,
+        "tpp_threshold_2_intended_diff_only": 0.023000001907348633,
+        "tpp_threshold_2_unintended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_5_total_metric": 0.018999993801116943,
+        "tpp_threshold_5_intended_diff_only": 0.018999993801116943,
+        "tpp_threshold_5_unintended_diff_only": 0.0,
+        "tpp_threshold_10_total_metric": 0.06325003504753113,
+        "tpp_threshold_10_intended_diff_only": 0.06700003147125244,
+        "tpp_threshold_10_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_20_total_metric": 0.0962500125169754,
+        "tpp_threshold_20_intended_diff_only": 0.10600000619888306,
+        "tpp_threshold_20_unintended_diff_only": 0.009749993681907654,
+        "tpp_threshold_50_total_metric": 0.25325004756450653,
+        "tpp_threshold_50_intended_diff_only": 0.2710000276565552,
+        "tpp_threshold_50_unintended_diff_only": 0.017749980092048645,
+        "tpp_threshold_100_total_metric": 0.31349998712539673,
+        "tpp_threshold_100_intended_diff_only": 0.3330000042915344,
+        "tpp_threshold_100_unintended_diff_only": 0.019500017166137695,
+        "tpp_threshold_500_total_metric": 0.3425000458955765,
+        "tpp_threshold_500_intended_diff_only": 0.3760000467300415,
+        "tpp_threshold_500_unintended_diff_only": 0.03350000083446503
+      }
+    }
+  }
+}

random_seed_eval_results/tpp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_1_custom_sae_eval_results.json

@@ -0,0 +1,414 @@
+{
+  "eval_type_id": "tpp",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": false,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "096e7204-f9f3-43f1-9a8f-eeaba02309e8",
+  "datetime_epoch_millis": 1738793209531,
+  "eval_result_metrics": {
+    "tpp_metrics": {
+      "tpp_threshold_2_total_metric": 0.010400001704692841,
+      "tpp_threshold_2_intended_diff_only": 0.013500005006790161,
+      "tpp_threshold_2_unintended_diff_only": 0.0031000033020973207,
+      "tpp_threshold_5_total_metric": 0.01885000616312027,
+      "tpp_threshold_5_intended_diff_only": 0.022200006246566772,
+      "tpp_threshold_5_unintended_diff_only": 0.003350000083446503,
+      "tpp_threshold_10_total_metric": 0.02705000340938568,
+      "tpp_threshold_10_intended_diff_only": 0.03130000233650208,
+      "tpp_threshold_10_unintended_diff_only": 0.004249998927116394,
+      "tpp_threshold_20_total_metric": 0.05490000545978546,
+      "tpp_threshold_20_intended_diff_only": 0.06270000338554382,
+      "tpp_threshold_20_unintended_diff_only": 0.007799997925758362,
+      "tpp_threshold_50_total_metric": 0.12465001344680787,
+      "tpp_threshold_50_intended_diff_only": 0.13350001573562623,
+      "tpp_threshold_50_unintended_diff_only": 0.008850002288818359,
+      "tpp_threshold_100_total_metric": 0.22405002117156983,
+      "tpp_threshold_100_intended_diff_only": 0.233400022983551,
+      "tpp_threshold_100_unintended_diff_only": 0.009350001811981201,
+      "tpp_threshold_500_total_metric": 0.40177501887083056,
+      "tpp_threshold_500_intended_diff_only": 0.419100022315979,
+      "tpp_threshold_500_unintended_diff_only": 0.01732500344514847
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
+      "tpp_threshold_2_total_metric": 0.010649994015693665,
+      "tpp_threshold_2_intended_diff_only": 0.012999999523162841,
+      "tpp_threshold_2_unintended_diff_only": 0.0023500055074691774,
+      "tpp_threshold_5_total_metric": 0.01860001087188721,
+      "tpp_threshold_5_intended_diff_only": 0.02120000123977661,
+      "tpp_threshold_5_unintended_diff_only": 0.002599990367889404,
+      "tpp_threshold_10_total_metric": 0.030500003695487977,
+      "tpp_threshold_10_intended_diff_only": 0.033399999141693115,
+      "tpp_threshold_10_unintended_diff_only": 0.002899995446205139,
+      "tpp_threshold_20_total_metric": 0.06360001266002654,
+      "tpp_threshold_20_intended_diff_only": 0.07040001153945923,
+      "tpp_threshold_20_unintended_diff_only": 0.006799998879432678,
+      "tpp_threshold_50_total_metric": 0.13365001678466798,
+      "tpp_threshold_50_intended_diff_only": 0.1414000153541565,
+      "tpp_threshold_50_unintended_diff_only": 0.007749998569488525,
+      "tpp_threshold_100_total_metric": 0.264050030708313,
+      "tpp_threshold_100_intended_diff_only": 0.27040002346038816,
+      "tpp_threshold_100_unintended_diff_only": 0.006349992752075195,
+      "tpp_threshold_500_total_metric": 0.4458000212907791,
+      "tpp_threshold_500_intended_diff_only": 0.45540002584457395,
+      "tpp_threshold_500_unintended_diff_only": 0.009600004553794861
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
+      "tpp_threshold_2_total_metric": 0.010150009393692016,
+      "tpp_threshold_2_intended_diff_only": 0.014000010490417481,
+      "tpp_threshold_2_unintended_diff_only": 0.003850001096725464,
+      "tpp_threshold_5_total_metric": 0.01910000145435333,
+      "tpp_threshold_5_intended_diff_only": 0.023200011253356932,
+      "tpp_threshold_5_unintended_diff_only": 0.004100009799003601,
+      "tpp_threshold_10_total_metric": 0.023600003123283385,
+      "tpp_threshold_10_intended_diff_only": 0.029200005531311034,
+      "tpp_threshold_10_unintended_diff_only": 0.005600002408027649,
+      "tpp_threshold_20_total_metric": 0.04619999825954437,
+      "tpp_threshold_20_intended_diff_only": 0.05499999523162842,
+      "tpp_threshold_20_unintended_diff_only": 0.008799996972084046,
+      "tpp_threshold_50_total_metric": 0.11565001010894775,
+      "tpp_threshold_50_intended_diff_only": 0.12560001611709595,
+      "tpp_threshold_50_unintended_diff_only": 0.009950006008148193,
+      "tpp_threshold_100_total_metric": 0.18405001163482665,
+      "tpp_threshold_100_intended_diff_only": 0.19640002250671387,
+      "tpp_threshold_100_unintended_diff_only": 0.012350010871887206,
+      "tpp_threshold_500_total_metric": 0.35775001645088195,
+      "tpp_threshold_500_intended_diff_only": 0.382800018787384,
+      "tpp_threshold_500_unintended_diff_only": 0.025050002336502075
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_1",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1": {
+      "0": {
+        "tpp_threshold_2_total_metric": 0.005750000476837158,
+        "tpp_threshold_2_intended_diff_only": 0.009000003337860107,
+        "tpp_threshold_2_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_5_total_metric": 0.012000024318695068,
+        "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
+        "tpp_threshold_5_unintended_diff_only": 0.0009999871253967285,
+        "tpp_threshold_10_total_metric": 0.021500051021575928,
+        "tpp_threshold_10_intended_diff_only": 0.021000027656555176,
+        "tpp_threshold_10_unintended_diff_only": -0.000500023365020752,
+        "tpp_threshold_20_total_metric": 0.02525004744529724,
+        "tpp_threshold_20_intended_diff_only": 0.029000043869018555,
+        "tpp_threshold_20_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_50_total_metric": 0.06125004589557648,
+        "tpp_threshold_50_intended_diff_only": 0.06700003147125244,
+        "tpp_threshold_50_unintended_diff_only": 0.005749985575675964,
+        "tpp_threshold_100_total_metric": 0.1480000615119934,
+        "tpp_threshold_100_intended_diff_only": 0.15000003576278687,
+        "tpp_threshold_100_unintended_diff_only": 0.001999974250793457,
+        "tpp_threshold_500_total_metric": 0.4255000650882721,
+        "tpp_threshold_500_intended_diff_only": 0.43000006675720215,
+        "tpp_threshold_500_unintended_diff_only": 0.004500001668930054
+      },
+      "1": {
+        "tpp_threshold_2_total_metric": 0.006749972701072693,
+        "tpp_threshold_2_intended_diff_only": 0.010999977588653564,
+        "tpp_threshold_2_unintended_diff_only": 0.004250004887580872,
+        "tpp_threshold_5_total_metric": -0.0017499923706054688,
+        "tpp_threshold_5_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_5_unintended_diff_only": 0.005750000476837158,
+        "tpp_threshold_10_total_metric": 0.010749995708465576,
+        "tpp_threshold_10_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_20_total_metric": 0.019000008702278137,
+        "tpp_threshold_20_intended_diff_only": 0.023000001907348633,
+        "tpp_threshold_20_unintended_diff_only": 0.003999993205070496,
+        "tpp_threshold_50_total_metric": 0.04249997437000275,
+        "tpp_threshold_50_intended_diff_only": 0.04799997806549072,
+        "tpp_threshold_50_unintended_diff_only": 0.005500003695487976,
+        "tpp_threshold_100_total_metric": 0.20250001549720764,
+        "tpp_threshold_100_intended_diff_only": 0.2070000171661377,
+        "tpp_threshold_100_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_500_total_metric": 0.43525002896785736,
+        "tpp_threshold_500_intended_diff_only": 0.44700002670288086,
+        "tpp_threshold_500_unintended_diff_only": 0.011749997735023499
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.02275000512599945,
+        "tpp_threshold_2_intended_diff_only": 0.027000010013580322,
+        "tpp_threshold_2_unintended_diff_only": 0.004250004887580872,
+        "tpp_threshold_5_total_metric": 0.04124997556209564,
+        "tpp_threshold_5_intended_diff_only": 0.042999982833862305,
+        "tpp_threshold_5_unintended_diff_only": 0.0017500072717666626,
+        "tpp_threshold_10_total_metric": 0.04849998652935028,
+        "tpp_threshold_10_intended_diff_only": 0.05199998617172241,
+        "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313,
+        "tpp_threshold_20_total_metric": 0.07100000977516174,
+        "tpp_threshold_20_intended_diff_only": 0.07400000095367432,
+        "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732,
+        "tpp_threshold_50_total_metric": 0.1365000307559967,
+        "tpp_threshold_50_intended_diff_only": 0.14100003242492676,
+        "tpp_threshold_50_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_100_total_metric": 0.22975002229213715,
+        "tpp_threshold_100_intended_diff_only": 0.23400002717971802,
+        "tpp_threshold_100_unintended_diff_only": 0.004250004887580872,
+        "tpp_threshold_500_total_metric": 0.4334999918937683,
+        "tpp_threshold_500_intended_diff_only": 0.4440000057220459,
+        "tpp_threshold_500_unintended_diff_only": 0.010500013828277588
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.003000035881996155,
+        "tpp_threshold_2_intended_diff_only": 0.0020000338554382324,
+        "tpp_threshold_2_unintended_diff_only": -0.0010000020265579224,
+        "tpp_threshold_5_total_metric": 0.003250017762184143,
+        "tpp_threshold_5_intended_diff_only": 0.004999995231628418,
+        "tpp_threshold_5_unintended_diff_only": 0.001749977469444275,
+        "tpp_threshold_10_total_metric": 0.007750004529953003,
+        "tpp_threshold_10_intended_diff_only": 0.009000003337860107,
+        "tpp_threshold_10_unintended_diff_only": 0.0012499988079071045,
+        "tpp_threshold_20_total_metric": 0.10650002956390381,
+        "tpp_threshold_20_intended_diff_only": 0.12200003862380981,
+        "tpp_threshold_20_unintended_diff_only": 0.015500009059906006,
+        "tpp_threshold_50_total_metric": 0.2407500445842743,
+        "tpp_threshold_50_intended_diff_only": 0.2560000419616699,
+        "tpp_threshold_50_unintended_diff_only": 0.01524999737739563,
+        "tpp_threshold_100_total_metric": 0.37400004267692566,
+        "tpp_threshold_100_intended_diff_only": 0.38700002431869507,
+        "tpp_threshold_100_unintended_diff_only": 0.01299998164176941,
+        "tpp_threshold_500_total_metric": 0.4647500365972519,
+        "tpp_threshold_500_intended_diff_only": 0.4790000319480896,
+        "tpp_threshold_500_unintended_diff_only": 0.014249995350837708
+      },
+      "9": {
+        "tpp_threshold_2_total_metric": 0.014999955892562866,
+        "tpp_threshold_2_intended_diff_only": 0.015999972820281982,
+        "tpp_threshold_2_unintended_diff_only": 0.0010000169277191162,
+        "tpp_threshold_5_total_metric": 0.03825002908706665,
+        "tpp_threshold_5_intended_diff_only": 0.04100000858306885,
+        "tpp_threshold_5_unintended_diff_only": 0.0027499794960021973,
+        "tpp_threshold_10_total_metric": 0.06399998068809509,
+        "tpp_threshold_10_intended_diff_only": 0.07099997997283936,
+        "tpp_threshold_10_unintended_diff_only": 0.006999999284744263,
+        "tpp_threshold_20_total_metric": 0.09624996781349182,
+        "tpp_threshold_20_intended_diff_only": 0.10399997234344482,
+        "tpp_threshold_20_unintended_diff_only": 0.007750004529953003,
+        "tpp_threshold_50_total_metric": 0.18724998831748962,
+        "tpp_threshold_50_intended_diff_only": 0.19499999284744263,
+        "tpp_threshold_50_unintended_diff_only": 0.007750004529953003,
+        "tpp_threshold_100_total_metric": 0.3660000115633011,
+        "tpp_threshold_100_intended_diff_only": 0.37400001287460327,
+        "tpp_threshold_100_unintended_diff_only": 0.008000001311302185,
+        "tpp_threshold_500_total_metric": 0.4699999839067459,
+        "tpp_threshold_500_intended_diff_only": 0.47699999809265137,
+        "tpp_threshold_500_unintended_diff_only": 0.0070000141859054565
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5": {
+      "1": {
+        "tpp_threshold_2_total_metric": 0.006250053644180298,
+        "tpp_threshold_2_intended_diff_only": 0.010000050067901611,
+        "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_5_total_metric": 0.007500022649765015,
+        "tpp_threshold_5_intended_diff_only": 0.012000024318695068,
+        "tpp_threshold_5_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_10_total_metric": 0.010000020265579224,
+        "tpp_threshold_10_intended_diff_only": 0.017000019550323486,
+        "tpp_threshold_10_unintended_diff_only": 0.006999999284744263,
+        "tpp_threshold_20_total_metric": 0.011750027537345886,
+        "tpp_threshold_20_intended_diff_only": 0.022000014781951904,
+        "tpp_threshold_20_unintended_diff_only": 0.010249987244606018,
+        "tpp_threshold_50_total_metric": 0.034000054001808167,
+        "tpp_threshold_50_intended_diff_only": 0.04300004243850708,
+        "tpp_threshold_50_unintended_diff_only": 0.008999988436698914,
+        "tpp_threshold_100_total_metric": 0.07725003361701965,
+        "tpp_threshold_100_intended_diff_only": 0.08500003814697266,
+        "tpp_threshold_100_unintended_diff_only": 0.007750004529953003,
+        "tpp_threshold_500_total_metric": 0.34775005280971527,
+        "tpp_threshold_500_intended_diff_only": 0.35700005292892456,
+        "tpp_threshold_500_unintended_diff_only": 0.00925000011920929
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.012499943375587463,
+        "tpp_threshold_2_intended_diff_only": 0.01699995994567871,
+        "tpp_threshold_2_unintended_diff_only": 0.0045000165700912476,
+        "tpp_threshold_5_total_metric": 0.019499972462654114,
+        "tpp_threshold_5_intended_diff_only": 0.02399998903274536,
+        "tpp_threshold_5_unintended_diff_only": 0.0045000165700912476,
+        "tpp_threshold_10_total_metric": 0.0207500159740448,
+        "tpp_threshold_10_intended_diff_only": 0.027000010013580322,
+        "tpp_threshold_10_unintended_diff_only": 0.0062499940395355225,
+        "tpp_threshold_20_total_metric": 0.0469999760389328,
+        "tpp_threshold_20_intended_diff_only": 0.05699998140335083,
+        "tpp_threshold_20_unintended_diff_only": 0.01000000536441803,
+        "tpp_threshold_50_total_metric": 0.11425000429153442,
+        "tpp_threshold_50_intended_diff_only": 0.12400001287460327,
+        "tpp_threshold_50_unintended_diff_only": 0.009750008583068848,
+        "tpp_threshold_100_total_metric": 0.2214999794960022,
+        "tpp_threshold_100_intended_diff_only": 0.23600000143051147,
+        "tpp_threshold_100_unintended_diff_only": 0.014500021934509277,
+        "tpp_threshold_500_total_metric": 0.39799998700618744,
+        "tpp_threshold_500_intended_diff_only": 0.4269999861717224,
+        "tpp_threshold_500_unintended_diff_only": 0.028999999165534973
+      },
+      "3": {
+        "tpp_threshold_2_total_metric": 0.002749994397163391,
+        "tpp_threshold_2_intended_diff_only": 0.0009999871253967285,
+        "tpp_threshold_2_unintended_diff_only": -0.0017500072717666626,
+        "tpp_threshold_5_total_metric": -0.0015000253915786743,
+        "tpp_threshold_5_intended_diff_only": 0.0009999871253967285,
+        "tpp_threshold_5_unintended_diff_only": 0.002500012516975403,
+        "tpp_threshold_10_total_metric": -0.00025004148483276367,
+        "tpp_threshold_10_intended_diff_only": 0.0029999613761901855,
+        "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_20_total_metric": 0.014499962329864502,
+        "tpp_threshold_20_intended_diff_only": 0.0209999680519104,
+        "tpp_threshold_20_unintended_diff_only": 0.0065000057220458984,
+        "tpp_threshold_50_total_metric": 0.04299996793270111,
+        "tpp_threshold_50_intended_diff_only": 0.05199998617172241,
+        "tpp_threshold_50_unintended_diff_only": 0.009000018239021301,
+        "tpp_threshold_100_total_metric": 0.08699999749660492,
+        "tpp_threshold_100_intended_diff_only": 0.09600001573562622,
+        "tpp_threshold_100_unintended_diff_only": 0.009000018239021301,
+        "tpp_threshold_500_total_metric": 0.3267500102519989,
+        "tpp_threshold_500_intended_diff_only": 0.3540000319480896,
+        "tpp_threshold_500_unintended_diff_only": 0.027250021696090698
+      },
+      "5": {
+        "tpp_threshold_2_total_metric": 0.015250042080879211,
+        "tpp_threshold_2_intended_diff_only": 0.025000035762786865,
+        "tpp_threshold_2_unintended_diff_only": 0.009749993681907654,
+        "tpp_threshold_5_total_metric": 0.028000012040138245,
+        "tpp_threshold_5_intended_diff_only": 0.0350000262260437,
+        "tpp_threshold_5_unintended_diff_only": 0.0070000141859054565,
+        "tpp_threshold_10_total_metric": 0.04300001263618469,
+        "tpp_threshold_10_intended_diff_only": 0.04900002479553223,
+        "tpp_threshold_10_unintended_diff_only": 0.006000012159347534,
+        "tpp_threshold_20_total_metric": 0.0780000239610672,
+        "tpp_threshold_20_intended_diff_only": 0.0910000205039978,
+        "tpp_threshold_20_unintended_diff_only": 0.012999996542930603,
+        "tpp_threshold_50_total_metric": 0.1822500377893448,
+        "tpp_threshold_50_intended_diff_only": 0.19600003957748413,
+        "tpp_threshold_50_unintended_diff_only": 0.013750001788139343,
+        "tpp_threshold_100_total_metric": 0.23400002717971802,
+        "tpp_threshold_100_intended_diff_only": 0.25200003385543823,
+        "tpp_threshold_100_unintended_diff_only": 0.018000006675720215,
+        "tpp_threshold_500_total_metric": 0.36400002241134644,
+        "tpp_threshold_500_intended_diff_only": 0.4020000100135803,
+        "tpp_threshold_500_unintended_diff_only": 0.03799998760223389
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.01400001347064972,
+        "tpp_threshold_2_intended_diff_only": 0.017000019550323486,
+        "tpp_threshold_2_unintended_diff_only": 0.003000006079673767,
+        "tpp_threshold_5_total_metric": 0.042000025510787964,
+        "tpp_threshold_5_intended_diff_only": 0.04400002956390381,
+        "tpp_threshold_5_unintended_diff_only": 0.0020000040531158447,
+        "tpp_threshold_10_total_metric": 0.04450000822544098,
+        "tpp_threshold_10_intended_diff_only": 0.050000011920928955,
+        "tpp_threshold_10_unintended_diff_only": 0.005500003695487976,
+        "tpp_threshold_20_total_metric": 0.07975000143051147,
+        "tpp_threshold_20_intended_diff_only": 0.08399999141693115,
+        "tpp_threshold_20_unintended_diff_only": 0.004249989986419678,
+        "tpp_threshold_50_total_metric": 0.20474998652935028,
+        "tpp_threshold_50_intended_diff_only": 0.21299999952316284,
+        "tpp_threshold_50_unintended_diff_only": 0.008250012993812561,
+        "tpp_threshold_100_total_metric": 0.3005000203847885,
+        "tpp_threshold_100_intended_diff_only": 0.31300002336502075,
+        "tpp_threshold_100_unintended_diff_only": 0.012500002980232239,
+        "tpp_threshold_500_total_metric": 0.35225000977516174,
+        "tpp_threshold_500_intended_diff_only": 0.37400001287460327,
+        "tpp_threshold_500_unintended_diff_only": 0.02175000309944153
+      }
+    }
+  }
+}

random_seed_eval_results/tpp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_2_custom_sae_eval_results.json

@@ -0,0 +1,414 @@
+{
+  "eval_type_id": "tpp",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": false,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "b73c894a-3e5b-4138-b75b-ceb8d5a28bdd",
+  "datetime_epoch_millis": 1738793438068,
+  "eval_result_metrics": {
+    "tpp_metrics": {
+      "tpp_threshold_2_total_metric": 0.010875004529953002,
+      "tpp_threshold_2_intended_diff_only": 0.014300006628036498,
+      "tpp_threshold_2_unintended_diff_only": 0.003425002098083496,
+      "tpp_threshold_5_total_metric": 0.017374998331069945,
+      "tpp_threshold_5_intended_diff_only": 0.021299999952316285,
+      "tpp_threshold_5_unintended_diff_only": 0.003925001621246338,
+      "tpp_threshold_10_total_metric": 0.032375001907348634,
+      "tpp_threshold_10_intended_diff_only": 0.03820000290870666,
+      "tpp_threshold_10_unintended_diff_only": 0.0058250010013580315,
+      "tpp_threshold_20_total_metric": 0.06415000408887864,
+      "tpp_threshold_20_intended_diff_only": 0.07130000591278077,
+      "tpp_threshold_20_unintended_diff_only": 0.00715000182390213,
+      "tpp_threshold_50_total_metric": 0.1440250039100647,
+      "tpp_threshold_50_intended_diff_only": 0.1531000077724457,
+      "tpp_threshold_50_unintended_diff_only": 0.00907500386238098,
+      "tpp_threshold_100_total_metric": 0.2260250121355057,
+      "tpp_threshold_100_intended_diff_only": 0.23930001258850098,
+      "tpp_threshold_100_unintended_diff_only": 0.0132750004529953,
+      "tpp_threshold_500_total_metric": 0.40262500792741773,
+      "tpp_threshold_500_intended_diff_only": 0.4224000096321106,
+      "tpp_threshold_500_unintended_diff_only": 0.01977500170469284
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
+      "tpp_threshold_2_total_metric": 0.007649996876716613,
+      "tpp_threshold_2_intended_diff_only": 0.010399997234344482,
+      "tpp_threshold_2_unintended_diff_only": 0.0027500003576278686,
+      "tpp_threshold_5_total_metric": 0.018599998950958253,
+      "tpp_threshold_5_intended_diff_only": 0.022200000286102296,
+      "tpp_threshold_5_unintended_diff_only": 0.003600001335144043,
+      "tpp_threshold_10_total_metric": 0.034799987077713014,
+      "tpp_threshold_10_intended_diff_only": 0.040999984741210936,
+      "tpp_threshold_10_unintended_diff_only": 0.006199997663497925,
+      "tpp_threshold_20_total_metric": 0.08539999425411224,
+      "tpp_threshold_20_intended_diff_only": 0.09199999570846558,
+      "tpp_threshold_20_unintended_diff_only": 0.006600001454353332,
+      "tpp_threshold_50_total_metric": 0.16370000541210175,
+      "tpp_threshold_50_intended_diff_only": 0.17060000896453859,
+      "tpp_threshold_50_unintended_diff_only": 0.006900003552436829,
+      "tpp_threshold_100_total_metric": 0.26725001335144044,
+      "tpp_threshold_100_intended_diff_only": 0.27660001516342164,
+      "tpp_threshold_100_unintended_diff_only": 0.009350001811981201,
+      "tpp_threshold_500_total_metric": 0.44620001316070557,
+      "tpp_threshold_500_intended_diff_only": 0.45620001554489137,
+      "tpp_threshold_500_unintended_diff_only": 0.010000002384185792
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
+      "tpp_threshold_2_total_metric": 0.014100012183189393,
+      "tpp_threshold_2_intended_diff_only": 0.018200016021728514,
+      "tpp_threshold_2_unintended_diff_only": 0.004100003838539123,
+      "tpp_threshold_5_total_metric": 0.01614999771118164,
+      "tpp_threshold_5_intended_diff_only": 0.020399999618530274,
+      "tpp_threshold_5_unintended_diff_only": 0.004250001907348633,
+      "tpp_threshold_10_total_metric": 0.029950016736984254,
+      "tpp_threshold_10_intended_diff_only": 0.035400021076202395,
+      "tpp_threshold_10_unintended_diff_only": 0.005450004339218139,
+      "tpp_threshold_20_total_metric": 0.04290001392364502,
+      "tpp_threshold_20_intended_diff_only": 0.05060001611709595,
+      "tpp_threshold_20_unintended_diff_only": 0.007700002193450928,
+      "tpp_threshold_50_total_metric": 0.12435000240802765,
+      "tpp_threshold_50_intended_diff_only": 0.1356000065803528,
+      "tpp_threshold_50_unintended_diff_only": 0.011250004172325134,
+      "tpp_threshold_100_total_metric": 0.18480001091957093,
+      "tpp_threshold_100_intended_diff_only": 0.2020000100135803,
+      "tpp_threshold_100_unintended_diff_only": 0.017199999094009398,
+      "tpp_threshold_500_total_metric": 0.35905000269412995,
+      "tpp_threshold_500_intended_diff_only": 0.38860000371932985,
+      "tpp_threshold_500_unintended_diff_only": 0.02955000102519989
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_2",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1": {
+      "0": {
+        "tpp_threshold_2_total_metric": 0.01250004768371582,
+        "tpp_threshold_2_intended_diff_only": 0.01500004529953003,
+        "tpp_threshold_2_unintended_diff_only": 0.002499997615814209,
+        "tpp_threshold_5_total_metric": 0.016500040888786316,
+        "tpp_threshold_5_intended_diff_only": 0.020000040531158447,
+        "tpp_threshold_5_unintended_diff_only": 0.0034999996423721313,
+        "tpp_threshold_10_total_metric": 0.014999985694885254,
+        "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
+        "tpp_threshold_10_unintended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_20_total_metric": 0.030500024557113647,
+        "tpp_threshold_20_intended_diff_only": 0.0350000262260437,
+        "tpp_threshold_20_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_50_total_metric": 0.05800001323223114,
+        "tpp_threshold_50_intended_diff_only": 0.06300002336502075,
+        "tpp_threshold_50_unintended_diff_only": 0.005000010132789612,
+        "tpp_threshold_100_total_metric": 0.1482500582933426,
+        "tpp_threshold_100_intended_diff_only": 0.15400004386901855,
+        "tpp_threshold_100_unintended_diff_only": 0.005749985575675964,
+        "tpp_threshold_500_total_metric": 0.43025003373622894,
+        "tpp_threshold_500_intended_diff_only": 0.43300002813339233,
+        "tpp_threshold_500_unintended_diff_only": 0.002749994397163391
+      },
+      "1": {
+        "tpp_threshold_2_total_metric": 0.006249964237213135,
+        "tpp_threshold_2_intended_diff_only": 0.006999969482421875,
+        "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402,
+        "tpp_threshold_5_total_metric": 0.003999963402748108,
+        "tpp_threshold_5_intended_diff_only": 0.006999969482421875,
+        "tpp_threshold_5_unintended_diff_only": 0.003000006079673767,
+        "tpp_threshold_10_total_metric": 0.007999956607818604,
+        "tpp_threshold_10_intended_diff_only": 0.007999956607818604,
+        "tpp_threshold_10_unintended_diff_only": 0.0,
+        "tpp_threshold_20_total_metric": 0.02549995481967926,
+        "tpp_threshold_20_intended_diff_only": 0.030999958515167236,
+        "tpp_threshold_20_unintended_diff_only": 0.005500003695487976,
+        "tpp_threshold_50_total_metric": 0.06399998068809509,
+        "tpp_threshold_50_intended_diff_only": 0.07099997997283936,
+        "tpp_threshold_50_unintended_diff_only": 0.006999999284744263,
+        "tpp_threshold_100_total_metric": 0.19199995696544647,
+        "tpp_threshold_100_intended_diff_only": 0.20099997520446777,
+        "tpp_threshold_100_unintended_diff_only": 0.009000018239021301,
+        "tpp_threshold_500_total_metric": 0.4364999681711197,
+        "tpp_threshold_500_intended_diff_only": 0.44599997997283936,
+        "tpp_threshold_500_unintended_diff_only": 0.009500011801719666
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.010749980807304382,
+        "tpp_threshold_2_intended_diff_only": 0.014999985694885254,
+        "tpp_threshold_2_unintended_diff_only": 0.004250004887580872,
+        "tpp_threshold_5_total_metric": 0.02699999511241913,
+        "tpp_threshold_5_intended_diff_only": 0.03299999237060547,
+        "tpp_threshold_5_unintended_diff_only": 0.00599999725818634,
+        "tpp_threshold_10_total_metric": 0.05600002408027649,
+        "tpp_threshold_10_intended_diff_only": 0.05900001525878906,
+        "tpp_threshold_10_unintended_diff_only": 0.0029999911785125732,
+        "tpp_threshold_20_total_metric": 0.07649999856948853,
+        "tpp_threshold_20_intended_diff_only": 0.07899999618530273,
+        "tpp_threshold_20_unintended_diff_only": 0.002499997615814209,
+        "tpp_threshold_50_total_metric": 0.14124999940395355,
+        "tpp_threshold_50_intended_diff_only": 0.14300000667572021,
+        "tpp_threshold_50_unintended_diff_only": 0.0017500072717666626,
+        "tpp_threshold_100_total_metric": 0.23274999856948853,
+        "tpp_threshold_100_intended_diff_only": 0.23600000143051147,
+        "tpp_threshold_100_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_500_total_metric": 0.4345000237226486,
+        "tpp_threshold_500_intended_diff_only": 0.44300001859664917,
+        "tpp_threshold_500_unintended_diff_only": 0.00849999487400055
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.0020000040531158447,
+        "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_2_unintended_diff_only": 0.0020000040531158447,
+        "tpp_threshold_5_total_metric": 0.008250012993812561,
+        "tpp_threshold_5_intended_diff_only": 0.008000016212463379,
+        "tpp_threshold_5_unintended_diff_only": -0.00024999678134918213,
+        "tpp_threshold_10_total_metric": 0.03300000727176666,
+        "tpp_threshold_10_intended_diff_only": 0.050999999046325684,
+        "tpp_threshold_10_unintended_diff_only": 0.01799999177455902,
+        "tpp_threshold_20_total_metric": 0.19350002706050873,
+        "tpp_threshold_20_intended_diff_only": 0.20600003004074097,
+        "tpp_threshold_20_unintended_diff_only": 0.012500002980232239,
+        "tpp_threshold_50_total_metric": 0.3450000137090683,
+        "tpp_threshold_50_intended_diff_only": 0.36000001430511475,
+        "tpp_threshold_50_unintended_diff_only": 0.015000000596046448,
+        "tpp_threshold_100_total_metric": 0.3972500413656235,
+        "tpp_threshold_100_intended_diff_only": 0.4140000343322754,
+        "tpp_threshold_100_unintended_diff_only": 0.016749992966651917,
+        "tpp_threshold_500_total_metric": 0.4632500112056732,
+        "tpp_threshold_500_intended_diff_only": 0.48000001907348633,
+        "tpp_threshold_500_unintended_diff_only": 0.01675000786781311
+      },
+      "9": {
+        "tpp_threshold_2_total_metric": 0.006749987602233887,
+        "tpp_threshold_2_intended_diff_only": 0.010999977588653564,
+        "tpp_threshold_2_unintended_diff_only": 0.004249989986419678,
+        "tpp_threshold_5_total_metric": 0.037249982357025146,
+        "tpp_threshold_5_intended_diff_only": 0.042999982833862305,
+        "tpp_threshold_5_unintended_diff_only": 0.005750000476837158,
+        "tpp_threshold_10_total_metric": 0.061999961733818054,
+        "tpp_threshold_10_intended_diff_only": 0.0679999589920044,
+        "tpp_threshold_10_unintended_diff_only": 0.00599999725818634,
+        "tpp_threshold_20_total_metric": 0.10099996626377106,
+        "tpp_threshold_20_intended_diff_only": 0.10899996757507324,
+        "tpp_threshold_20_unintended_diff_only": 0.008000001311302185,
+        "tpp_threshold_50_total_metric": 0.21025002002716064,
+        "tpp_threshold_50_intended_diff_only": 0.2160000205039978,
+        "tpp_threshold_50_unintended_diff_only": 0.005750000476837158,
+        "tpp_threshold_100_total_metric": 0.3660000115633011,
+        "tpp_threshold_100_intended_diff_only": 0.37800002098083496,
+        "tpp_threshold_100_unintended_diff_only": 0.012000009417533875,
+        "tpp_threshold_500_total_metric": 0.46650002896785736,
+        "tpp_threshold_500_intended_diff_only": 0.4790000319480896,
+        "tpp_threshold_500_unintended_diff_only": 0.012500002980232239
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5": {
+      "1": {
+        "tpp_threshold_2_total_metric": 0.012250036001205444,
+        "tpp_threshold_2_intended_diff_only": 0.016000032424926758,
+        "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_5_total_metric": 0.010250017046928406,
+        "tpp_threshold_5_intended_diff_only": 0.013000011444091797,
+        "tpp_threshold_5_unintended_diff_only": 0.002749994397163391,
+        "tpp_threshold_10_total_metric": 0.009500056505203247,
+        "tpp_threshold_10_intended_diff_only": 0.01900005340576172,
+        "tpp_threshold_10_unintended_diff_only": 0.009499996900558472,
+        "tpp_threshold_20_total_metric": 0.02825005352497101,
+        "tpp_threshold_20_intended_diff_only": 0.03400003910064697,
+        "tpp_threshold_20_unintended_diff_only": 0.005749985575675964,
+        "tpp_threshold_50_total_metric": 0.06874999403953552,
+        "tpp_threshold_50_intended_diff_only": 0.07400000095367432,
+        "tpp_threshold_50_unintended_diff_only": 0.005250006914138794,
+        "tpp_threshold_100_total_metric": 0.1260000467300415,
+        "tpp_threshold_100_intended_diff_only": 0.14000004529953003,
+        "tpp_threshold_100_unintended_diff_only": 0.013999998569488525,
+        "tpp_threshold_500_total_metric": 0.38225002586841583,
+        "tpp_threshold_500_intended_diff_only": 0.3970000147819519,
+        "tpp_threshold_500_unintended_diff_only": 0.014749988913536072
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.018999993801116943,
+        "tpp_threshold_2_intended_diff_only": 0.023000001907348633,
+        "tpp_threshold_2_unintended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_5_total_metric": 0.014999955892562866,
+        "tpp_threshold_5_intended_diff_only": 0.02599996328353882,
+        "tpp_threshold_5_unintended_diff_only": 0.011000007390975952,
+        "tpp_threshold_10_total_metric": 0.027500003576278687,
+        "tpp_threshold_10_intended_diff_only": 0.03200000524520874,
+        "tpp_threshold_10_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_20_total_metric": 0.0352499783039093,
+        "tpp_threshold_20_intended_diff_only": 0.041999995708465576,
+        "tpp_threshold_20_unintended_diff_only": 0.006750017404556274,
+        "tpp_threshold_50_total_metric": 0.09974999725818634,
+        "tpp_threshold_50_intended_diff_only": 0.11500000953674316,
+        "tpp_threshold_50_unintended_diff_only": 0.015250012278556824,
+        "tpp_threshold_100_total_metric": 0.16974999010562897,
+        "tpp_threshold_100_intended_diff_only": 0.1899999976158142,
+        "tpp_threshold_100_unintended_diff_only": 0.02025000751018524,
+        "tpp_threshold_500_total_metric": 0.38850001990795135,
+        "tpp_threshold_500_intended_diff_only": 0.42000001668930054,
+        "tpp_threshold_500_unintended_diff_only": 0.03149999678134918
+      },
+      "3": {
+        "tpp_threshold_2_total_metric": -0.008750006556510925,
+        "tpp_threshold_2_intended_diff_only": -0.004999995231628418,
+        "tpp_threshold_2_unintended_diff_only": 0.0037500113248825073,
+        "tpp_threshold_5_total_metric": -0.0015000104904174805,
+        "tpp_threshold_5_intended_diff_only": 0.0,
+        "tpp_threshold_5_unintended_diff_only": 0.0015000104904174805,
+        "tpp_threshold_10_total_metric": 0.01649998128414154,
+        "tpp_threshold_10_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_10_unintended_diff_only": 0.0015000253915786743,
+        "tpp_threshold_20_total_metric": 0.00849999487400055,
+        "tpp_threshold_20_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_20_unintended_diff_only": 0.009500011801719666,
+        "tpp_threshold_50_total_metric": 0.05700001120567322,
+        "tpp_threshold_50_intended_diff_only": 0.0690000057220459,
+        "tpp_threshold_50_unintended_diff_only": 0.01199999451637268,
+        "tpp_threshold_100_total_metric": 0.10050001740455627,
+        "tpp_threshold_100_intended_diff_only": 0.11900001764297485,
+        "tpp_threshold_100_unintended_diff_only": 0.01850000023841858,
+        "tpp_threshold_500_total_metric": 0.3189999610185623,
+        "tpp_threshold_500_intended_diff_only": 0.3529999852180481,
+        "tpp_threshold_500_unintended_diff_only": 0.03400002419948578
+      },
+      "5": {
+        "tpp_threshold_2_total_metric": 0.02550002932548523,
+        "tpp_threshold_2_intended_diff_only": 0.030000030994415283,
+        "tpp_threshold_2_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_5_total_metric": 0.029000014066696167,
+        "tpp_threshold_5_intended_diff_only": 0.03600001335144043,
+        "tpp_threshold_5_unintended_diff_only": 0.006999999284744263,
+        "tpp_threshold_10_total_metric": 0.03625001013278961,
+        "tpp_threshold_10_intended_diff_only": 0.046000003814697266,
+        "tpp_threshold_10_unintended_diff_only": 0.009749993681907654,
+        "tpp_threshold_20_total_metric": 0.04950001835823059,
+        "tpp_threshold_20_intended_diff_only": 0.06000000238418579,
+        "tpp_threshold_20_unintended_diff_only": 0.0104999840259552,
+        "tpp_threshold_50_total_metric": 0.18125002086162567,
+        "tpp_threshold_50_intended_diff_only": 0.19300001859664917,
+        "tpp_threshold_50_unintended_diff_only": 0.011749997735023499,
+        "tpp_threshold_100_total_metric": 0.22349999845027924,
+        "tpp_threshold_100_intended_diff_only": 0.2409999966621399,
+        "tpp_threshold_100_unintended_diff_only": 0.017499998211860657,
+        "tpp_threshold_500_total_metric": 0.35450001060962677,
+        "tpp_threshold_500_intended_diff_only": 0.39800000190734863,
+        "tpp_threshold_500_unintended_diff_only": 0.04349999129772186
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.02250000834465027,
+        "tpp_threshold_2_intended_diff_only": 0.027000010013580322,
+        "tpp_threshold_2_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_5_total_metric": 0.028000012040138245,
+        "tpp_threshold_5_intended_diff_only": 0.027000010013580322,
+        "tpp_threshold_5_unintended_diff_only": -0.0010000020265579224,
+        "tpp_threshold_10_total_metric": 0.06000003218650818,
+        "tpp_threshold_10_intended_diff_only": 0.06200003623962402,
+        "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447,
+        "tpp_threshold_20_total_metric": 0.09300002455711365,
+        "tpp_threshold_20_intended_diff_only": 0.09900003671646118,
+        "tpp_threshold_20_unintended_diff_only": 0.006000012159347534,
+        "tpp_threshold_50_total_metric": 0.2149999886751175,
+        "tpp_threshold_50_intended_diff_only": 0.22699999809265137,
+        "tpp_threshold_50_unintended_diff_only": 0.012000009417533875,
+        "tpp_threshold_100_total_metric": 0.30425000190734863,
+        "tpp_threshold_100_intended_diff_only": 0.3199999928474426,
+        "tpp_threshold_100_unintended_diff_only": 0.015749990940093994,
+        "tpp_threshold_500_total_metric": 0.35099999606609344,
+        "tpp_threshold_500_intended_diff_only": 0.375,
+        "tpp_threshold_500_unintended_diff_only": 0.024000003933906555
+      }
+    }
+  }
+}

random_seed_eval_results/tpp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_3_custom_sae_eval_results.json

@@ -0,0 +1,414 @@
+{
+  "eval_type_id": "tpp",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": false,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "4c97b22e-3ce7-4f44-9382-ba43c6b1f096",
+  "datetime_epoch_millis": 1738793552532,
+  "eval_result_metrics": {
+    "tpp_metrics": {
+      "tpp_threshold_2_total_metric": 0.007999995350837707,
+      "tpp_threshold_2_intended_diff_only": 0.011000001430511476,
+      "tpp_threshold_2_unintended_diff_only": 0.003000006079673767,
+      "tpp_threshold_5_total_metric": 0.01237499564886093,
+      "tpp_threshold_5_intended_diff_only": 0.016099995374679564,
+      "tpp_threshold_5_unintended_diff_only": 0.003724999725818634,
+      "tpp_threshold_10_total_metric": 0.027025008201599122,
+      "tpp_threshold_10_intended_diff_only": 0.03130000829696655,
+      "tpp_threshold_10_unintended_diff_only": 0.004275000095367432,
+      "tpp_threshold_20_total_metric": 0.05402499288320541,
+      "tpp_threshold_20_intended_diff_only": 0.060099995136260985,
+      "tpp_threshold_20_unintended_diff_only": 0.0060750022530555725,
+      "tpp_threshold_50_total_metric": 0.12652500867843627,
+      "tpp_threshold_50_intended_diff_only": 0.13470001220703126,
+      "tpp_threshold_50_unintended_diff_only": 0.00817500352859497,
+      "tpp_threshold_100_total_metric": 0.2042750060558319,
+      "tpp_threshold_100_intended_diff_only": 0.2156000018119812,
+      "tpp_threshold_100_unintended_diff_only": 0.011324995756149292,
+      "tpp_threshold_500_total_metric": 0.3991500198841095,
+      "tpp_threshold_500_intended_diff_only": 0.41680002212524414,
+      "tpp_threshold_500_unintended_diff_only": 0.017650002241134645
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
+      "tpp_threshold_2_total_metric": 0.007299986481666565,
+      "tpp_threshold_2_intended_diff_only": 0.00979999303817749,
+      "tpp_threshold_2_unintended_diff_only": 0.0025000065565109254,
+      "tpp_threshold_5_total_metric": 0.010649988055229187,
+      "tpp_threshold_5_intended_diff_only": 0.013799989223480224,
+      "tpp_threshold_5_unintended_diff_only": 0.0031500011682510376,
+      "tpp_threshold_10_total_metric": 0.022449997067451478,
+      "tpp_threshold_10_intended_diff_only": 0.025199997425079345,
+      "tpp_threshold_10_unintended_diff_only": 0.0027500003576278686,
+      "tpp_threshold_20_total_metric": 0.060399994254112244,
+      "tpp_threshold_20_intended_diff_only": 0.06639999151229858,
+      "tpp_threshold_20_unintended_diff_only": 0.00599999725818634,
+      "tpp_threshold_50_total_metric": 0.13545000851154326,
+      "tpp_threshold_50_intended_diff_only": 0.14160001277923584,
+      "tpp_threshold_50_unintended_diff_only": 0.006150004267692566,
+      "tpp_threshold_100_total_metric": 0.23460001051425933,
+      "tpp_threshold_100_intended_diff_only": 0.24240000247955323,
+      "tpp_threshold_100_unintended_diff_only": 0.007799991965293884,
+      "tpp_threshold_500_total_metric": 0.4378500312566757,
+      "tpp_threshold_500_intended_diff_only": 0.4468000292778015,
+      "tpp_threshold_500_unintended_diff_only": 0.008949998021125793
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
+      "tpp_threshold_2_total_metric": 0.00870000422000885,
+      "tpp_threshold_2_intended_diff_only": 0.01220000982284546,
+      "tpp_threshold_2_unintended_diff_only": 0.003500005602836609,
+      "tpp_threshold_5_total_metric": 0.014100003242492675,
+      "tpp_threshold_5_intended_diff_only": 0.018400001525878906,
+      "tpp_threshold_5_unintended_diff_only": 0.0042999982833862305,
+      "tpp_threshold_10_total_metric": 0.031600019335746764,
+      "tpp_threshold_10_intended_diff_only": 0.03740001916885376,
+      "tpp_threshold_10_unintended_diff_only": 0.005799999833106995,
+      "tpp_threshold_20_total_metric": 0.04764999151229858,
+      "tpp_threshold_20_intended_diff_only": 0.053799998760223386,
+      "tpp_threshold_20_unintended_diff_only": 0.006150007247924805,
+      "tpp_threshold_50_total_metric": 0.11760000884532928,
+      "tpp_threshold_50_intended_diff_only": 0.12780001163482665,
+      "tpp_threshold_50_unintended_diff_only": 0.010200002789497375,
+      "tpp_threshold_100_total_metric": 0.17395000159740448,
+      "tpp_threshold_100_intended_diff_only": 0.18880000114440917,
+      "tpp_threshold_100_unintended_diff_only": 0.0148499995470047,
+      "tpp_threshold_500_total_metric": 0.3604500085115433,
+      "tpp_threshold_500_intended_diff_only": 0.3868000149726868,
+      "tpp_threshold_500_unintended_diff_only": 0.026350006461143494
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_3",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1": {
+      "0": {
+        "tpp_threshold_2_total_metric": 0.011749997735023499,
+        "tpp_threshold_2_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_2_unintended_diff_only": 0.002250000834465027,
+        "tpp_threshold_5_total_metric": 0.014500007033348083,
+        "tpp_threshold_5_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_5_unintended_diff_only": 0.0034999996423721313,
+        "tpp_threshold_10_total_metric": 0.014999985694885254,
+        "tpp_threshold_10_intended_diff_only": 0.018999993801116943,
+        "tpp_threshold_10_unintended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_20_total_metric": 0.034000009298324585,
+        "tpp_threshold_20_intended_diff_only": 0.03700000047683716,
+        "tpp_threshold_20_unintended_diff_only": 0.0029999911785125732,
+        "tpp_threshold_50_total_metric": 0.05224999785423279,
+        "tpp_threshold_50_intended_diff_only": 0.0559999942779541,
+        "tpp_threshold_50_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_100_total_metric": 0.1365000456571579,
+        "tpp_threshold_100_intended_diff_only": 0.14000004529953003,
+        "tpp_threshold_100_unintended_diff_only": 0.0034999996423721313,
+        "tpp_threshold_500_total_metric": 0.42850005626678467,
+        "tpp_threshold_500_intended_diff_only": 0.4320000410079956,
+        "tpp_threshold_500_unintended_diff_only": 0.0034999847412109375
+      },
+      "1": {
+        "tpp_threshold_2_total_metric": 0.001749977469444275,
+        "tpp_threshold_2_intended_diff_only": 0.001999974250793457,
+        "tpp_threshold_2_unintended_diff_only": 0.00024999678134918213,
+        "tpp_threshold_5_total_metric": -0.0027500689029693604,
+        "tpp_threshold_5_intended_diff_only": -0.001000046730041504,
+        "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564,
+        "tpp_threshold_10_total_metric": 0.0027499794960021973,
+        "tpp_threshold_10_intended_diff_only": 0.001999974250793457,
+        "tpp_threshold_10_unintended_diff_only": -0.0007500052452087402,
+        "tpp_threshold_20_total_metric": 0.0260000079870224,
+        "tpp_threshold_20_intended_diff_only": 0.03200000524520874,
+        "tpp_threshold_20_unintended_diff_only": 0.00599999725818634,
+        "tpp_threshold_50_total_metric": 0.10000000894069672,
+        "tpp_threshold_50_intended_diff_only": 0.10600000619888306,
+        "tpp_threshold_50_unintended_diff_only": 0.00599999725818634,
+        "tpp_threshold_100_total_metric": 0.16649998724460602,
+        "tpp_threshold_100_intended_diff_only": 0.17299997806549072,
+        "tpp_threshold_100_unintended_diff_only": 0.006499990820884705,
+        "tpp_threshold_500_total_metric": 0.3929999768733978,
+        "tpp_threshold_500_intended_diff_only": 0.3999999761581421,
+        "tpp_threshold_500_unintended_diff_only": 0.006999999284744263
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.00974997878074646,
+        "tpp_threshold_2_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_2_unintended_diff_only": 0.004250019788742065,
+        "tpp_threshold_5_total_metric": 0.01899997889995575,
+        "tpp_threshold_5_intended_diff_only": 0.02399998903274536,
+        "tpp_threshold_5_unintended_diff_only": 0.005000010132789612,
+        "tpp_threshold_10_total_metric": 0.032250016927719116,
+        "tpp_threshold_10_intended_diff_only": 0.0350000262260437,
+        "tpp_threshold_10_unintended_diff_only": 0.002750009298324585,
+        "tpp_threshold_20_total_metric": 0.05024999380111694,
+        "tpp_threshold_20_intended_diff_only": 0.05199998617172241,
+        "tpp_threshold_20_unintended_diff_only": 0.0017499923706054688,
+        "tpp_threshold_50_total_metric": 0.0884999930858612,
+        "tpp_threshold_50_intended_diff_only": 0.08799999952316284,
+        "tpp_threshold_50_unintended_diff_only": -0.0004999935626983643,
+        "tpp_threshold_100_total_metric": 0.1704999953508377,
+        "tpp_threshold_100_intended_diff_only": 0.17299997806549072,
+        "tpp_threshold_100_unintended_diff_only": 0.002499982714653015,
+        "tpp_threshold_500_total_metric": 0.43925003707408905,
+        "tpp_threshold_500_intended_diff_only": 0.44600003957748413,
+        "tpp_threshold_500_unintended_diff_only": 0.006750002503395081
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.002499997615814209,
+        "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_2_unintended_diff_only": 0.0015000104904174805,
+        "tpp_threshold_5_total_metric": 0.0025000572204589844,
+        "tpp_threshold_5_intended_diff_only": 0.0020000338554382324,
+        "tpp_threshold_5_unintended_diff_only": -0.000500023365020752,
+        "tpp_threshold_10_total_metric": 0.006750002503395081,
+        "tpp_threshold_10_intended_diff_only": 0.009000003337860107,
+        "tpp_threshold_10_unintended_diff_only": 0.002250000834465027,
+        "tpp_threshold_20_total_metric": 0.09875001013278961,
+        "tpp_threshold_20_intended_diff_only": 0.11100000143051147,
+        "tpp_threshold_20_unintended_diff_only": 0.012249991297721863,
+        "tpp_threshold_50_total_metric": 0.24550004303455353,
+        "tpp_threshold_50_intended_diff_only": 0.2600000500679016,
+        "tpp_threshold_50_unintended_diff_only": 0.014500007033348083,
+        "tpp_threshold_100_total_metric": 0.3797500282526016,
+        "tpp_threshold_100_intended_diff_only": 0.3960000276565552,
+        "tpp_threshold_100_unintended_diff_only": 0.016249999403953552,
+        "tpp_threshold_500_total_metric": 0.46150006353855133,
+        "tpp_threshold_500_intended_diff_only": 0.47700005769729614,
+        "tpp_threshold_500_unintended_diff_only": 0.015499994158744812
+      },
+      "9": {
+        "tpp_threshold_2_total_metric": 0.010749980807304382,
+        "tpp_threshold_2_intended_diff_only": 0.014999985694885254,
+        "tpp_threshold_2_unintended_diff_only": 0.004250004887580872,
+        "tpp_threshold_5_total_metric": 0.019999966025352478,
+        "tpp_threshold_5_intended_diff_only": 0.02599996328353882,
+        "tpp_threshold_5_unintended_diff_only": 0.00599999725818634,
+        "tpp_threshold_10_total_metric": 0.05550000071525574,
+        "tpp_threshold_10_intended_diff_only": 0.06099998950958252,
+        "tpp_threshold_10_unintended_diff_only": 0.005499988794326782,
+        "tpp_threshold_20_total_metric": 0.09299995005130768,
+        "tpp_threshold_20_intended_diff_only": 0.09999996423721313,
+        "tpp_threshold_20_unintended_diff_only": 0.0070000141859054565,
+        "tpp_threshold_50_total_metric": 0.19099999964237213,
+        "tpp_threshold_50_intended_diff_only": 0.1980000138282776,
+        "tpp_threshold_50_unintended_diff_only": 0.0070000141859054565,
+        "tpp_threshold_100_total_metric": 0.31974999606609344,
+        "tpp_threshold_100_intended_diff_only": 0.32999998331069946,
+        "tpp_threshold_100_unintended_diff_only": 0.010249987244606018,
+        "tpp_threshold_500_total_metric": 0.4670000225305557,
+        "tpp_threshold_500_intended_diff_only": 0.4790000319480896,
+        "tpp_threshold_500_unintended_diff_only": 0.012000009417533875
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5": {
+      "1": {
+        "tpp_threshold_2_total_metric": 0.010499998927116394,
+        "tpp_threshold_2_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_2_unintended_diff_only": 0.0034999996423721313,
+        "tpp_threshold_5_total_metric": 0.01075001060962677,
+        "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_5_unintended_diff_only": 0.0032499879598617554,
+        "tpp_threshold_10_total_metric": 0.011500045657157898,
+        "tpp_threshold_10_intended_diff_only": 0.020000040531158447,
+        "tpp_threshold_10_unintended_diff_only": 0.00849999487400055,
+        "tpp_threshold_20_total_metric": 0.022750049829483032,
+        "tpp_threshold_20_intended_diff_only": 0.029000043869018555,
+        "tpp_threshold_20_unintended_diff_only": 0.0062499940395355225,
+        "tpp_threshold_50_total_metric": 0.0585000216960907,
+        "tpp_threshold_50_intended_diff_only": 0.06300002336502075,
+        "tpp_threshold_50_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_100_total_metric": 0.09350000321865082,
+        "tpp_threshold_100_intended_diff_only": 0.10600000619888306,
+        "tpp_threshold_100_unintended_diff_only": 0.012500002980232239,
+        "tpp_threshold_500_total_metric": 0.37125004827976227,
+        "tpp_threshold_500_intended_diff_only": 0.3840000629425049,
+        "tpp_threshold_500_unintended_diff_only": 0.012750014662742615
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.016249999403953552,
+        "tpp_threshold_2_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_2_unintended_diff_only": 0.0017500072717666626,
+        "tpp_threshold_5_total_metric": 0.01099996268749237,
+        "tpp_threshold_5_intended_diff_only": 0.0209999680519104,
+        "tpp_threshold_5_unintended_diff_only": 0.01000000536441803,
+        "tpp_threshold_10_total_metric": 0.03600001335144043,
+        "tpp_threshold_10_intended_diff_only": 0.04100000858306885,
+        "tpp_threshold_10_unintended_diff_only": 0.004999995231628418,
+        "tpp_threshold_20_total_metric": 0.05199997127056122,
+        "tpp_threshold_20_intended_diff_only": 0.05699998140335083,
+        "tpp_threshold_20_unintended_diff_only": 0.005000010132789612,
+        "tpp_threshold_50_total_metric": 0.10375002026557922,
+        "tpp_threshold_50_intended_diff_only": 0.11500000953674316,
+        "tpp_threshold_50_unintended_diff_only": 0.01124998927116394,
+        "tpp_threshold_100_total_metric": 0.1720000058412552,
+        "tpp_threshold_100_intended_diff_only": 0.18900001049041748,
+        "tpp_threshold_100_unintended_diff_only": 0.017000004649162292,
+        "tpp_threshold_500_total_metric": 0.39124996960163116,
+        "tpp_threshold_500_intended_diff_only": 0.4179999828338623,
+        "tpp_threshold_500_unintended_diff_only": 0.02675001323223114
+      },
+      "3": {
+        "tpp_threshold_2_total_metric": -0.007500022649765015,
+        "tpp_threshold_2_intended_diff_only": -0.0040000081062316895,
+        "tpp_threshold_2_unintended_diff_only": 0.003500014543533325,
+        "tpp_threshold_5_total_metric": -0.0005000084638595581,
+        "tpp_threshold_5_intended_diff_only": 0.0,
+        "tpp_threshold_5_unintended_diff_only": 0.0005000084638595581,
+        "tpp_threshold_10_total_metric": 0.01299998164176941,
+        "tpp_threshold_10_intended_diff_only": 0.014999985694885254,
+        "tpp_threshold_10_unintended_diff_only": 0.0020000040531158447,
+        "tpp_threshold_20_total_metric": 0.005999967455863953,
+        "tpp_threshold_20_intended_diff_only": 0.010999977588653564,
+        "tpp_threshold_20_unintended_diff_only": 0.005000010132789612,
+        "tpp_threshold_50_total_metric": 0.057499960064888,
+        "tpp_threshold_50_intended_diff_only": 0.06699997186660767,
+        "tpp_threshold_50_unintended_diff_only": 0.009500011801719666,
+        "tpp_threshold_100_total_metric": 0.10624997317790985,
+        "tpp_threshold_100_intended_diff_only": 0.11799997091293335,
+        "tpp_threshold_100_unintended_diff_only": 0.011749997735023499,
+        "tpp_threshold_500_total_metric": 0.32750001549720764,
+        "tpp_threshold_500_intended_diff_only": 0.35500001907348633,
+        "tpp_threshold_500_unintended_diff_only": 0.027500003576278687
+      },
+      "5": {
+        "tpp_threshold_2_total_metric": 0.012000009417533875,
+        "tpp_threshold_2_intended_diff_only": 0.017000019550323486,
+        "tpp_threshold_2_unintended_diff_only": 0.005000010132789612,
+        "tpp_threshold_5_total_metric": 0.016750037670135498,
+        "tpp_threshold_5_intended_diff_only": 0.025000035762786865,
+        "tpp_threshold_5_unintended_diff_only": 0.008249998092651367,
+        "tpp_threshold_10_total_metric": 0.03800003230571747,
+        "tpp_threshold_10_intended_diff_only": 0.0480000376701355,
+        "tpp_threshold_10_unintended_diff_only": 0.01000000536441803,
+        "tpp_threshold_20_total_metric": 0.07374997437000275,
+        "tpp_threshold_20_intended_diff_only": 0.08399999141693115,
+        "tpp_threshold_20_unintended_diff_only": 0.010250017046928406,
+        "tpp_threshold_50_total_metric": 0.16600003838539124,
+        "tpp_threshold_50_intended_diff_only": 0.1770000457763672,
+        "tpp_threshold_50_unintended_diff_only": 0.011000007390975952,
+        "tpp_threshold_100_total_metric": 0.2147499918937683,
+        "tpp_threshold_100_intended_diff_only": 0.23199999332427979,
+        "tpp_threshold_100_unintended_diff_only": 0.017250001430511475,
+        "tpp_threshold_500_total_metric": 0.36249999701976776,
+        "tpp_threshold_500_intended_diff_only": 0.4020000100135803,
+        "tpp_threshold_500_unintended_diff_only": 0.03950001299381256
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.012250036001205444,
+        "tpp_threshold_2_intended_diff_only": 0.016000032424926758,
+        "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_5_total_metric": 0.0325000137090683,
+        "tpp_threshold_5_intended_diff_only": 0.03200000524520874,
+        "tpp_threshold_5_unintended_diff_only": -0.0005000084638595581,
+        "tpp_threshold_10_total_metric": 0.05950002372264862,
+        "tpp_threshold_10_intended_diff_only": 0.06300002336502075,
+        "tpp_threshold_10_unintended_diff_only": 0.0034999996423721313,
+        "tpp_threshold_20_total_metric": 0.08374999463558197,
+        "tpp_threshold_20_intended_diff_only": 0.08799999952316284,
+        "tpp_threshold_20_unintended_diff_only": 0.004250004887580872,
+        "tpp_threshold_50_total_metric": 0.20225000381469727,
+        "tpp_threshold_50_intended_diff_only": 0.21700000762939453,
+        "tpp_threshold_50_unintended_diff_only": 0.014750003814697266,
+        "tpp_threshold_100_total_metric": 0.28325003385543823,
+        "tpp_threshold_100_intended_diff_only": 0.2990000247955322,
+        "tpp_threshold_100_unintended_diff_only": 0.015749990940093994,
+        "tpp_threshold_500_total_metric": 0.34975001215934753,
+        "tpp_threshold_500_intended_diff_only": 0.375,
+        "tpp_threshold_500_unintended_diff_only": 0.025249987840652466
+      }
+    }
+  }
+}

random_seed_eval_results/tpp/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_4_custom_sae_eval_results.json

@@ -0,0 +1,414 @@
+{
+  "eval_type_id": "tpp",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "LabHC/bias_in_bios_class_set1",
+      "canrager/amazon_reviews_mcauley_1and5"
+    ],
+    "perform_scr": false,
+    "early_stopping_patience": 20,
+    "train_set_size": 4000,
+    "test_set_size": 1000,
+    "context_length": 128,
+    "probe_train_batch_size": 16,
+    "probe_test_batch_size": 500,
+    "probe_epochs": 20,
+    "probe_lr": 0.001,
+    "probe_l1_penalty": 0.001,
+    "sae_batch_size": 125,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "lower_vram_usage": false,
+    "model_name": "gemma-2-2b",
+    "n_values": [
+      2,
+      5,
+      10,
+      20,
+      50,
+      100,
+      500
+    ],
+    "column1_vals_lookup": {
+      "LabHC/bias_in_bios_class_set1": [
+        [
+          "professor",
+          "nurse"
+        ],
+        [
+          "architect",
+          "journalist"
+        ],
+        [
+          "surgeon",
+          "psychologist"
+        ],
+        [
+          "attorney",
+          "teacher"
+        ]
+      ],
+      "canrager/amazon_reviews_mcauley_1and5": [
+        [
+          "Books",
+          "CDs_and_Vinyl"
+        ],
+        [
+          "Software",
+          "Electronics"
+        ],
+        [
+          "Pet_Supplies",
+          "Office_Products"
+        ],
+        [
+          "Industrial_and_Scientific",
+          "Toys_and_Games"
+        ]
+      ]
+    }
+  },
+  "eval_id": "4aec626d-a48c-4e98-b34f-fe6bc6f9eb13",
+  "datetime_epoch_millis": 1738793667846,
+  "eval_result_metrics": {
+    "tpp_metrics": {
+      "tpp_threshold_2_total_metric": 0.008275003731250764,
+      "tpp_threshold_2_intended_diff_only": 0.0112000048160553,
+      "tpp_threshold_2_unintended_diff_only": 0.002925001084804535,
+      "tpp_threshold_5_total_metric": 0.013749995827674865,
+      "tpp_threshold_5_intended_diff_only": 0.01729999780654907,
+      "tpp_threshold_5_unintended_diff_only": 0.003550001978874206,
+      "tpp_threshold_10_total_metric": 0.02792499363422394,
+      "tpp_threshold_10_intended_diff_only": 0.0328000009059906,
+      "tpp_threshold_10_unintended_diff_only": 0.004875007271766663,
+      "tpp_threshold_20_total_metric": 0.0562250018119812,
+      "tpp_threshold_20_intended_diff_only": 0.06310000419616699,
+      "tpp_threshold_20_unintended_diff_only": 0.006875002384185791,
+      "tpp_threshold_50_total_metric": 0.13492498844861983,
+      "tpp_threshold_50_intended_diff_only": 0.1437999963760376,
+      "tpp_threshold_50_unintended_diff_only": 0.008875007927417754,
+      "tpp_threshold_100_total_metric": 0.21179999709129332,
+      "tpp_threshold_100_intended_diff_only": 0.22450000047683716,
+      "tpp_threshold_100_unintended_diff_only": 0.012700003385543824,
+      "tpp_threshold_500_total_metric": 0.3962500214576721,
+      "tpp_threshold_500_intended_diff_only": 0.41480002403259275,
+      "tpp_threshold_500_unintended_diff_only": 0.018550002574920656
+    }
+  },
+  "eval_result_details": [
+    {
+      "dataset_name": "LabHC/bias_in_bios_class_set1_tpp_results",
+      "tpp_threshold_2_total_metric": 0.010100004076957703,
+      "tpp_threshold_2_intended_diff_only": 0.012400007247924805,
+      "tpp_threshold_2_unintended_diff_only": 0.002300003170967102,
+      "tpp_threshold_5_total_metric": 0.015050002932548523,
+      "tpp_threshold_5_intended_diff_only": 0.018000006675720215,
+      "tpp_threshold_5_unintended_diff_only": 0.002950003743171692,
+      "tpp_threshold_10_total_metric": 0.02779998779296875,
+      "tpp_threshold_10_intended_diff_only": 0.030799996852874757,
+      "tpp_threshold_10_unintended_diff_only": 0.003000009059906006,
+      "tpp_threshold_20_total_metric": 0.07204999625682831,
+      "tpp_threshold_20_intended_diff_only": 0.0787999987602234,
+      "tpp_threshold_20_unintended_diff_only": 0.006750002503395081,
+      "tpp_threshold_50_total_metric": 0.15629999041557313,
+      "tpp_threshold_50_intended_diff_only": 0.1631999969482422,
+      "tpp_threshold_50_unintended_diff_only": 0.006900006532669067,
+      "tpp_threshold_100_total_metric": 0.2554499953985214,
+      "tpp_threshold_100_intended_diff_only": 0.2641999959945679,
+      "tpp_threshold_100_unintended_diff_only": 0.008750000596046447,
+      "tpp_threshold_500_total_metric": 0.44135003089904784,
+      "tpp_threshold_500_intended_diff_only": 0.45200003385543824,
+      "tpp_threshold_500_unintended_diff_only": 0.01065000295639038
+    },
+    {
+      "dataset_name": "canrager/amazon_reviews_mcauley_1and5_tpp_results",
+      "tpp_threshold_2_total_metric": 0.006450003385543824,
+      "tpp_threshold_2_intended_diff_only": 0.010000002384185792,
+      "tpp_threshold_2_unintended_diff_only": 0.0035499989986419677,
+      "tpp_threshold_5_total_metric": 0.012449988722801208,
+      "tpp_threshold_5_intended_diff_only": 0.01659998893737793,
+      "tpp_threshold_5_unintended_diff_only": 0.004150000214576721,
+      "tpp_threshold_10_total_metric": 0.028049999475479127,
+      "tpp_threshold_10_intended_diff_only": 0.03480000495910644,
+      "tpp_threshold_10_unintended_diff_only": 0.0067500054836273195,
+      "tpp_threshold_20_total_metric": 0.040400007367134096,
+      "tpp_threshold_20_intended_diff_only": 0.04740000963211059,
+      "tpp_threshold_20_unintended_diff_only": 0.007000002264976502,
+      "tpp_threshold_50_total_metric": 0.11354998648166656,
+      "tpp_threshold_50_intended_diff_only": 0.12439999580383301,
+      "tpp_threshold_50_unintended_diff_only": 0.010850009322166444,
+      "tpp_threshold_100_total_metric": 0.16814999878406525,
+      "tpp_threshold_100_intended_diff_only": 0.18480000495910645,
+      "tpp_threshold_100_unintended_diff_only": 0.0166500061750412,
+      "tpp_threshold_500_total_metric": 0.35115001201629636,
+      "tpp_threshold_500_intended_diff_only": 0.3776000142097473,
+      "tpp_threshold_500_unintended_diff_only": 0.02645000219345093
+    }
+  ],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_4",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": {
+    "LabHC/bias_in_bios_class_set1": {
+      "0": {
+        "tpp_threshold_2_total_metric": 0.011749997735023499,
+        "tpp_threshold_2_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_2_unintended_diff_only": 0.002250000834465027,
+        "tpp_threshold_5_total_metric": 0.016250044107437134,
+        "tpp_threshold_5_intended_diff_only": 0.020000040531158447,
+        "tpp_threshold_5_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_10_total_metric": 0.013750001788139343,
+        "tpp_threshold_10_intended_diff_only": 0.017000019550323486,
+        "tpp_threshold_10_unintended_diff_only": 0.003250017762184143,
+        "tpp_threshold_20_total_metric": 0.02750001847743988,
+        "tpp_threshold_20_intended_diff_only": 0.03100001811981201,
+        "tpp_threshold_20_unintended_diff_only": 0.0034999996423721313,
+        "tpp_threshold_50_total_metric": 0.059250012040138245,
+        "tpp_threshold_50_intended_diff_only": 0.06300002336502075,
+        "tpp_threshold_50_unintended_diff_only": 0.0037500113248825073,
+        "tpp_threshold_100_total_metric": 0.12549999356269836,
+        "tpp_threshold_100_intended_diff_only": 0.12999999523162842,
+        "tpp_threshold_100_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_500_total_metric": 0.42475004494190216,
+        "tpp_threshold_500_intended_diff_only": 0.4270000457763672,
+        "tpp_threshold_500_unintended_diff_only": 0.002250000834465027
+      },
+      "1": {
+        "tpp_threshold_2_total_metric": 0.004250004887580872,
+        "tpp_threshold_2_intended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_2_unintended_diff_only": -0.00024999678134918213,
+        "tpp_threshold_5_total_metric": 0.001249939203262329,
+        "tpp_threshold_5_intended_diff_only": 0.0029999613761901855,
+        "tpp_threshold_5_unintended_diff_only": 0.0017500221729278564,
+        "tpp_threshold_10_total_metric": 0.009499981999397278,
+        "tpp_threshold_10_intended_diff_only": 0.009999990463256836,
+        "tpp_threshold_10_unintended_diff_only": 0.0005000084638595581,
+        "tpp_threshold_20_total_metric": 0.047999992966651917,
+        "tpp_threshold_20_intended_diff_only": 0.05500000715255737,
+        "tpp_threshold_20_unintended_diff_only": 0.0070000141859054565,
+        "tpp_threshold_50_total_metric": 0.14000000059604645,
+        "tpp_threshold_50_intended_diff_only": 0.14800000190734863,
+        "tpp_threshold_50_unintended_diff_only": 0.008000001311302185,
+        "tpp_threshold_100_total_metric": 0.22099995613098145,
+        "tpp_threshold_100_intended_diff_only": 0.22899997234344482,
+        "tpp_threshold_100_unintended_diff_only": 0.008000016212463379,
+        "tpp_threshold_500_total_metric": 0.435000017285347,
+        "tpp_threshold_500_intended_diff_only": 0.44700002670288086,
+        "tpp_threshold_500_unintended_diff_only": 0.012000009417533875
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.014749988913536072,
+        "tpp_threshold_2_intended_diff_only": 0.018999993801116943,
+        "tpp_threshold_2_unintended_diff_only": 0.004250004887580872,
+        "tpp_threshold_5_total_metric": 0.02250000834465027,
+        "tpp_threshold_5_intended_diff_only": 0.027000010013580322,
+        "tpp_threshold_5_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_10_total_metric": 0.043749988079071045,
+        "tpp_threshold_10_intended_diff_only": 0.046999990940093994,
+        "tpp_threshold_10_unintended_diff_only": 0.0032500028610229492,
+        "tpp_threshold_20_total_metric": 0.06425000727176666,
+        "tpp_threshold_20_intended_diff_only": 0.06499999761581421,
+        "tpp_threshold_20_unintended_diff_only": 0.0007499903440475464,
+        "tpp_threshold_50_total_metric": 0.11674997210502625,
+        "tpp_threshold_50_intended_diff_only": 0.11699998378753662,
+        "tpp_threshold_50_unintended_diff_only": 0.000250011682510376,
+        "tpp_threshold_100_total_metric": 0.18924999237060547,
+        "tpp_threshold_100_intended_diff_only": 0.19099998474121094,
+        "tpp_threshold_100_unintended_diff_only": 0.0017499923706054688,
+        "tpp_threshold_500_total_metric": 0.4217500239610672,
+        "tpp_threshold_500_intended_diff_only": 0.42900002002716064,
+        "tpp_threshold_500_unintended_diff_only": 0.007249996066093445
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.0022500157356262207,
+        "tpp_threshold_2_intended_diff_only": 0.003000020980834961,
+        "tpp_threshold_2_unintended_diff_only": 0.0007500052452087402,
+        "tpp_threshold_5_total_metric": 0.0027500540018081665,
+        "tpp_threshold_5_intended_diff_only": 0.0020000338554382324,
+        "tpp_threshold_5_unintended_diff_only": -0.0007500201463699341,
+        "tpp_threshold_10_total_metric": 0.009750023484230042,
+        "tpp_threshold_10_intended_diff_only": 0.012000024318695068,
+        "tpp_threshold_10_unintended_diff_only": 0.002250000834465027,
+        "tpp_threshold_20_total_metric": 0.11100000143051147,
+        "tpp_threshold_20_intended_diff_only": 0.125,
+        "tpp_threshold_20_unintended_diff_only": 0.013999998569488525,
+        "tpp_threshold_50_total_metric": 0.25349998474121094,
+        "tpp_threshold_50_intended_diff_only": 0.26899999380111694,
+        "tpp_threshold_50_unintended_diff_only": 0.015500009059906006,
+        "tpp_threshold_100_total_metric": 0.36350004374980927,
+        "tpp_threshold_100_intended_diff_only": 0.3810000419616699,
+        "tpp_threshold_100_unintended_diff_only": 0.017499998211860657,
+        "tpp_threshold_500_total_metric": 0.4607500433921814,
+        "tpp_threshold_500_intended_diff_only": 0.47800004482269287,
+        "tpp_threshold_500_unintended_diff_only": 0.017250001430511475
+      },
+      "9": {
+        "tpp_threshold_2_total_metric": 0.01750001311302185,
+        "tpp_threshold_2_intended_diff_only": 0.022000014781951904,
+        "tpp_threshold_2_unintended_diff_only": 0.004500001668930054,
+        "tpp_threshold_5_total_metric": 0.03249996900558472,
+        "tpp_threshold_5_intended_diff_only": 0.03799998760223389,
+        "tpp_threshold_5_unintended_diff_only": 0.00550001859664917,
+        "tpp_threshold_10_total_metric": 0.06224994361400604,
+        "tpp_threshold_10_intended_diff_only": 0.0679999589920044,
+        "tpp_threshold_10_unintended_diff_only": 0.005750015377998352,
+        "tpp_threshold_20_total_metric": 0.1094999611377716,
+        "tpp_threshold_20_intended_diff_only": 0.11799997091293335,
+        "tpp_threshold_20_unintended_diff_only": 0.008500009775161743,
+        "tpp_threshold_50_total_metric": 0.21199998259544373,
+        "tpp_threshold_50_intended_diff_only": 0.218999981880188,
+        "tpp_threshold_50_unintended_diff_only": 0.006999999284744263,
+        "tpp_threshold_100_total_metric": 0.3779999911785126,
+        "tpp_threshold_100_intended_diff_only": 0.38999998569488525,
+        "tpp_threshold_100_unintended_diff_only": 0.01199999451637268,
+        "tpp_threshold_500_total_metric": 0.4645000249147415,
+        "tpp_threshold_500_intended_diff_only": 0.4790000319480896,
+        "tpp_threshold_500_unintended_diff_only": 0.014500007033348083
+      }
+    },
+    "canrager/amazon_reviews_mcauley_1and5": {
+      "1": {
+        "tpp_threshold_2_total_metric": 0.008250027894973755,
+        "tpp_threshold_2_intended_diff_only": 0.012000024318695068,
+        "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_5_total_metric": 0.011000007390975952,
+        "tpp_threshold_5_intended_diff_only": 0.013999998569488525,
+        "tpp_threshold_5_unintended_diff_only": 0.0029999911785125732,
+        "tpp_threshold_10_total_metric": 0.009000018239021301,
+        "tpp_threshold_10_intended_diff_only": 0.017000019550323486,
+        "tpp_threshold_10_unintended_diff_only": 0.008000001311302185,
+        "tpp_threshold_20_total_metric": 0.01750001311302185,
+        "tpp_threshold_20_intended_diff_only": 0.023000001907348633,
+        "tpp_threshold_20_unintended_diff_only": 0.005499988794326782,
+        "tpp_threshold_50_total_metric": 0.05000004172325134,
+        "tpp_threshold_50_intended_diff_only": 0.05200004577636719,
+        "tpp_threshold_50_unintended_diff_only": 0.0020000040531158447,
+        "tpp_threshold_100_total_metric": 0.08300001919269562,
+        "tpp_threshold_100_intended_diff_only": 0.09200000762939453,
+        "tpp_threshold_100_unintended_diff_only": 0.008999988436698914,
+        "tpp_threshold_500_total_metric": 0.35725001990795135,
+        "tpp_threshold_500_intended_diff_only": 0.36900001764297485,
+        "tpp_threshold_500_unintended_diff_only": 0.011749997735023499
+      },
+      "2": {
+        "tpp_threshold_2_total_metric": 0.006999954581260681,
+        "tpp_threshold_2_intended_diff_only": 0.007999956607818604,
+        "tpp_threshold_2_unintended_diff_only": 0.0010000020265579224,
+        "tpp_threshold_5_total_metric": -0.0015000402927398682,
+        "tpp_threshold_5_intended_diff_only": 0.007999956607818604,
+        "tpp_threshold_5_unintended_diff_only": 0.009499996900558472,
+        "tpp_threshold_10_total_metric": 0.02675001323223114,
+        "tpp_threshold_10_intended_diff_only": 0.03600001335144043,
+        "tpp_threshold_10_unintended_diff_only": 0.00925000011920929,
+        "tpp_threshold_20_total_metric": 0.04025000333786011,
+        "tpp_threshold_20_intended_diff_only": 0.046000003814697266,
+        "tpp_threshold_20_unintended_diff_only": 0.005750000476837158,
+        "tpp_threshold_50_total_metric": 0.08999994397163391,
+        "tpp_threshold_50_intended_diff_only": 0.10499995946884155,
+        "tpp_threshold_50_unintended_diff_only": 0.015000015497207642,
+        "tpp_threshold_100_total_metric": 0.15524999797344208,
+        "tpp_threshold_100_intended_diff_only": 0.18000000715255737,
+        "tpp_threshold_100_unintended_diff_only": 0.024750009179115295,
+        "tpp_threshold_500_total_metric": 0.3857499957084656,
+        "tpp_threshold_500_intended_diff_only": 0.41200000047683716,
+        "tpp_threshold_500_unintended_diff_only": 0.026250004768371582
+      },
+      "3": {
+        "tpp_threshold_2_total_metric": -0.009999990463256836,
+        "tpp_threshold_2_intended_diff_only": -0.0059999823570251465,
+        "tpp_threshold_2_unintended_diff_only": 0.0040000081062316895,
+        "tpp_threshold_5_total_metric": 0.0007499754428863525,
+        "tpp_threshold_5_intended_diff_only": 0.0009999871253967285,
+        "tpp_threshold_5_unintended_diff_only": 0.000250011682510376,
+        "tpp_threshold_10_total_metric": 0.015499934554100037,
+        "tpp_threshold_10_intended_diff_only": 0.01699995994567871,
+        "tpp_threshold_10_unintended_diff_only": 0.0015000253915786743,
+        "tpp_threshold_20_total_metric": 0.004749968647956848,
+        "tpp_threshold_20_intended_diff_only": 0.014999985694885254,
+        "tpp_threshold_20_unintended_diff_only": 0.010250017046928406,
+        "tpp_threshold_50_total_metric": 0.0509999543428421,
+        "tpp_threshold_50_intended_diff_only": 0.06299996376037598,
+        "tpp_threshold_50_unintended_diff_only": 0.012000009417533875,
+        "tpp_threshold_100_total_metric": 0.09874998033046722,
+        "tpp_threshold_100_intended_diff_only": 0.1119999885559082,
+        "tpp_threshold_100_unintended_diff_only": 0.013250008225440979,
+        "tpp_threshold_500_total_metric": 0.31724995374679565,
+        "tpp_threshold_500_intended_diff_only": 0.343999981880188,
+        "tpp_threshold_500_unintended_diff_only": 0.026750028133392334
+      },
+      "5": {
+        "tpp_threshold_2_total_metric": 0.012750014662742615,
+        "tpp_threshold_2_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_2_unintended_diff_only": 0.0052499920129776,
+        "tpp_threshold_5_total_metric": 0.017499983310699463,
+        "tpp_threshold_5_intended_diff_only": 0.02399998903274536,
+        "tpp_threshold_5_unintended_diff_only": 0.0065000057220458984,
+        "tpp_threshold_10_total_metric": 0.02900002896785736,
+        "tpp_threshold_10_intended_diff_only": 0.03900003433227539,
+        "tpp_threshold_10_unintended_diff_only": 0.01000000536441803,
+        "tpp_threshold_20_total_metric": 0.04450002312660217,
+        "tpp_threshold_20_intended_diff_only": 0.053000032901763916,
+        "tpp_threshold_20_unintended_diff_only": 0.008500009775161743,
+        "tpp_threshold_50_total_metric": 0.15950000286102295,
+        "tpp_threshold_50_intended_diff_only": 0.17000001668930054,
+        "tpp_threshold_50_unintended_diff_only": 0.010500013828277588,
+        "tpp_threshold_100_total_metric": 0.21274997293949127,
+        "tpp_threshold_100_intended_diff_only": 0.23199999332427979,
+        "tpp_threshold_100_unintended_diff_only": 0.019250020384788513,
+        "tpp_threshold_500_total_metric": 0.346250057220459,
+        "tpp_threshold_500_intended_diff_only": 0.3890000581741333,
+        "tpp_threshold_500_unintended_diff_only": 0.042750000953674316
+      },
+      "6": {
+        "tpp_threshold_2_total_metric": 0.014250010251998901,
+        "tpp_threshold_2_intended_diff_only": 0.018000006675720215,
+        "tpp_threshold_2_unintended_diff_only": 0.0037499964237213135,
+        "tpp_threshold_5_total_metric": 0.03450001776218414,
+        "tpp_threshold_5_intended_diff_only": 0.03600001335144043,
+        "tpp_threshold_5_unintended_diff_only": 0.0014999955892562866,
+        "tpp_threshold_10_total_metric": 0.06000000238418579,
+        "tpp_threshold_10_intended_diff_only": 0.06499999761581421,
+        "tpp_threshold_10_unintended_diff_only": 0.004999995231628418,
+        "tpp_threshold_20_total_metric": 0.09500002861022949,
+        "tpp_threshold_20_intended_diff_only": 0.10000002384185791,
+        "tpp_threshold_20_unintended_diff_only": 0.004999995231628418,
+        "tpp_threshold_50_total_metric": 0.21724998950958252,
+        "tpp_threshold_50_intended_diff_only": 0.23199999332427979,
+        "tpp_threshold_50_unintended_diff_only": 0.014750003814697266,
+        "tpp_threshold_100_total_metric": 0.29100002348423004,
+        "tpp_threshold_100_intended_diff_only": 0.30800002813339233,
+        "tpp_threshold_100_unintended_diff_only": 0.017000004649162292,
+        "tpp_threshold_500_total_metric": 0.34925003349781036,
+        "tpp_threshold_500_intended_diff_only": 0.37400001287460327,
+        "tpp_threshold_500_unintended_diff_only": 0.024749979376792908
+      }
+    }
+  }
+}

random_seed_eval_results/unlearning/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_0_custom_sae_eval_results.json

@@ -0,0 +1,74 @@
+{
+  "eval_type_id": "unlearning",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "wmdp-bio",
+      "high_school_us_history",
+      "college_computer_science",
+      "high_school_geography",
+      "human_aging"
+    ],
+    "intervention_method": "clamp_feature_activation",
+    "retain_thresholds": [
+      0.001,
+      0.01
+    ],
+    "n_features_list": [
+      10,
+      20
+    ],
+    "multipliers": [
+      25,
+      50,
+      100,
+      200
+    ],
+    "dataset_size": 1024,
+    "seq_len": 1024,
+    "n_batch_loss_added": 50,
+    "target_metric": "correct",
+    "save_metrics": true,
+    "model_name": "gemma-2-2b-it",
+    "llm_batch_size": 4,
+    "llm_dtype": "bfloat16"
+  },
+  "eval_id": "8312f760-cdb0-4a9d-89f0-457797128bd3",
+  "datetime_epoch_millis": 1738798909354,
+  "eval_result_metrics": {
+    "unlearning": {
+      "unlearning_score": 0.24202626943588257
+    }
+  },
+  "eval_result_details": [],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_0",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

random_seed_eval_results/unlearning/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_1_custom_sae_eval_results.json

@@ -0,0 +1,74 @@
+{
+  "eval_type_id": "unlearning",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "wmdp-bio",
+      "high_school_us_history",
+      "college_computer_science",
+      "high_school_geography",
+      "human_aging"
+    ],
+    "intervention_method": "clamp_feature_activation",
+    "retain_thresholds": [
+      0.001,
+      0.01
+    ],
+    "n_features_list": [
+      10,
+      20
+    ],
+    "multipliers": [
+      25,
+      50,
+      100,
+      200
+    ],
+    "dataset_size": 1024,
+    "seq_len": 1024,
+    "n_batch_loss_added": 50,
+    "target_metric": "correct",
+    "save_metrics": true,
+    "model_name": "gemma-2-2b-it",
+    "llm_batch_size": 4,
+    "llm_dtype": "bfloat16"
+  },
+  "eval_id": "d16c8a20-cfa3-49b9-99d6-f7d41d99f940",
+  "datetime_epoch_millis": 1738798229725,
+  "eval_result_metrics": {
+    "unlearning": {
+      "unlearning_score": 0.056285202503204346
+    }
+  },
+  "eval_result_details": [],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_1",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

random_seed_eval_results/unlearning/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_2_custom_sae_eval_results.json

@@ -0,0 +1,74 @@
+{
+  "eval_type_id": "unlearning",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "wmdp-bio",
+      "high_school_us_history",
+      "college_computer_science",
+      "high_school_geography",
+      "human_aging"
+    ],
+    "intervention_method": "clamp_feature_activation",
+    "retain_thresholds": [
+      0.001,
+      0.01
+    ],
+    "n_features_list": [
+      10,
+      20
+    ],
+    "multipliers": [
+      25,
+      50,
+      100,
+      200
+    ],
+    "dataset_size": 1024,
+    "seq_len": 1024,
+    "n_batch_loss_added": 50,
+    "target_metric": "correct",
+    "save_metrics": true,
+    "model_name": "gemma-2-2b-it",
+    "llm_batch_size": 4,
+    "llm_dtype": "bfloat16"
+  },
+  "eval_id": "4497566c-7243-4b08-ac53-8e580570ae2c",
+  "datetime_epoch_millis": 1738799580619,
+  "eval_result_metrics": {
+    "unlearning": {
+      "unlearning_score": 0.0675421953201294
+    }
+  },
+  "eval_result_details": [],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_2",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

random_seed_eval_results/unlearning/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_3_custom_sae_eval_results.json

@@ -0,0 +1,74 @@
+{
+  "eval_type_id": "unlearning",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "wmdp-bio",
+      "high_school_us_history",
+      "college_computer_science",
+      "high_school_geography",
+      "human_aging"
+    ],
+    "intervention_method": "clamp_feature_activation",
+    "retain_thresholds": [
+      0.001,
+      0.01
+    ],
+    "n_features_list": [
+      10,
+      20
+    ],
+    "multipliers": [
+      25,
+      50,
+      100,
+      200
+    ],
+    "dataset_size": 1024,
+    "seq_len": 1024,
+    "n_batch_loss_added": 50,
+    "target_metric": "correct",
+    "save_metrics": true,
+    "model_name": "gemma-2-2b-it",
+    "llm_batch_size": 4,
+    "llm_dtype": "bfloat16"
+  },
+  "eval_id": "397e6fb4-1a07-4c16-9540-afc50a61875d",
+  "datetime_epoch_millis": 1738800255624,
+  "eval_result_metrics": {
+    "unlearning": {
+      "unlearning_score": 0.0731707215309143
+    }
+  },
+  "eval_result_details": [],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_3",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}

random_seed_eval_results/unlearning/temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_4_custom_sae_eval_results.json

@@ -0,0 +1,74 @@
+{
+  "eval_type_id": "unlearning",
+  "eval_config": {
+    "random_seed": 42,
+    "dataset_names": [
+      "wmdp-bio",
+      "high_school_us_history",
+      "college_computer_science",
+      "high_school_geography",
+      "human_aging"
+    ],
+    "intervention_method": "clamp_feature_activation",
+    "retain_thresholds": [
+      0.001,
+      0.01
+    ],
+    "n_features_list": [
+      10,
+      20
+    ],
+    "multipliers": [
+      25,
+      50,
+      100,
+      200
+    ],
+    "dataset_size": 1024,
+    "seq_len": 1024,
+    "n_batch_loss_added": 50,
+    "target_metric": "correct",
+    "save_metrics": true,
+    "model_name": "gemma-2-2b-it",
+    "llm_batch_size": 4,
+    "llm_dtype": "bfloat16"
+  },
+  "eval_id": "8ce47154-1e20-4092-b993-e151fed028b0",
+  "datetime_epoch_millis": 1738800936072,
+  "eval_result_metrics": {
+    "unlearning": {
+      "unlearning_score": 0.0731707215309143
+    }
+  },
+  "eval_result_details": [],
+  "sae_bench_commit_hash": "155afbca50a9ffe6cf72c81796997e6daa475658",
+  "sae_lens_id": "custom_sae",
+  "sae_lens_release_id": "temp_random_seeds_google_gemma-2-2b_top_k_resid_post_layer_12_trainer_4",
+  "sae_lens_version": "5.4.1",
+  "sae_cfg_dict": {
+    "model_name": "gemma-2-2b",
+    "d_in": 2304,
+    "d_sae": 16384,
+    "hook_layer": 12,
+    "hook_name": "blocks.12.hook_resid_post",
+    "context_size": null,
+    "hook_head_index": null,
+    "architecture": "topk",
+    "apply_b_dec_to_input": null,
+    "finetuning_scaling_factor": null,
+    "activation_fn_str": "",
+    "prepend_bos": true,
+    "normalize_activations": "none",
+    "dtype": "bfloat16",
+    "device": "",
+    "dataset_path": "",
+    "dataset_trust_remote_code": true,
+    "seqpos_slice": [
+      null
+    ],
+    "training_tokens": -100000,
+    "sae_lens_training_version": null,
+    "neuronpedia_id": null
+  },
+  "eval_result_unstructured": null
+}