800M-Slimpajama-0-OpenR1-Math-220k/L0R/config.json

{
    "sae_type": "sae",
    "hook_point_in": "blocks.0.hook_resid_post",
    "hook_point_out": "blocks.0.hook_resid_post",
    "d_model": 4096,
    "expansion_factor": 8,
    "use_decoder_bias": true,
    "use_glu_encoder": false,
    "act_fn": "jumprelu",
    "jump_relu_threshold": 0.0,
    "apply_decoder_bias_to_pre_encoder": false,
    "norm_activation": "dataset-wise",
    "sparsity_include_decoder_norm": false,
    "force_unit_decoder_norm": false,
    "top_k": 50,
    "use_triton_kernel": true,
    "sparsity_threshold_for_triton_spmm_kernel": 0.99,
    "jumprelu_threshold_window": 2.0
}