tangledgroup
/

tangled-llama-a-128k-base-v0.1

Text Generation

text-generation-inference

Inference Endpoints

Model card Files Files and versions Community

mtasic85 commited on 8 days ago

Commit

6cca977

•

1 Parent(s): bbec1b1

config

Files changed (1) hide show

config.json +9 -7

config.json CHANGED Viewed

@@ -1,22 +1,24 @@
 {
- "_name_or_path": "tangledgroup/tangled-llama-33m-32k-base-v0.1",
  "architectures": [
  "LlamaForCausalLM"
  ],
  "bos_token_id": 1,
  "eos_token_id": 2,
- "hidden_size": 1024,
- "intermediate_size": 4096,
- "max_position_embeddings": 38400,
  "model_type": "llama",
  "num_attention_heads": 32,
- "num_hidden_layers": 5,
  "num_key_value_heads": 8,
  "rms_norm_eps": 1e-05,
  "rope_scaling": null,
- "tie_word_embeddings": false,
  "torch_dtype": "bfloat16",
  "transformers_version": "4.44.2",
  "use_cache": true,
- "vocab_size": 38400
 }

 {
+ "_name_or_path": "tangledgroup/tangled-llama-a-32k-base-v0.1",
  "architectures": [
  "LlamaForCausalLM"
  ],
  "bos_token_id": 1,
  "eos_token_id": 2,
+ "head_dim": 64,
+ "hidden_act": "silu",
+ "hidden_size": 512,
+ "intermediate_size": 2048,
+ "max_position_embeddings": 8192,
  "model_type": "llama",
  "num_attention_heads": 32,
+ "num_hidden_layers": 8,
  "num_key_value_heads": 8,
  "rms_norm_eps": 1e-05,
  "rope_scaling": null,
+ "tie_word_embeddings": true,
  "torch_dtype": "bfloat16",
  "transformers_version": "4.44.2",
  "use_cache": true,
+ "vocab_size": 32768
 }