File size: 600 Bytes
9a59d23 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 |
{
"sae_type": "sae",
"hook_point_in": "blocks.0.hook_resid_post",
"hook_point_out": "blocks.0.hook_resid_post",
"d_model": 4096,
"expansion_factor": 8,
"use_decoder_bias": true,
"use_glu_encoder": false,
"act_fn": "jumprelu",
"jump_relu_threshold": 0.0,
"apply_decoder_bias_to_pre_encoder": false,
"norm_activation": "dataset-wise",
"sparsity_include_decoder_norm": false,
"force_unit_decoder_norm": false,
"top_k": 50,
"use_triton_kernel": true,
"sparsity_threshold_for_triton_spmm_kernel": 0.99,
"jumprelu_threshold_window": 2.0
} |