push model
Browse files- README.md +52 -0
- added_tokens.json +5 -0
- all_results.json +9 -0
- checkpoint-704/added_tokens.json +5 -0
- checkpoint-704/config.json +28 -0
- checkpoint-704/generation_config.json +6 -0
- checkpoint-704/merges.txt +0 -0
- checkpoint-704/model.safetensors +3 -0
- checkpoint-704/optimizer.pt +3 -0
- checkpoint-704/rng_state.pth +3 -0
- checkpoint-704/scheduler.pt +3 -0
- checkpoint-704/special_tokens_map.json +20 -0
- checkpoint-704/tokenizer.json +0 -0
- checkpoint-704/tokenizer_config.json +43 -0
- checkpoint-704/trainer_state.json +28 -0
- checkpoint-704/training_args.bin +3 -0
- checkpoint-704/vocab.json +0 -0
- checkpoint-720/added_tokens.json +5 -0
- checkpoint-720/config.json +28 -0
- checkpoint-720/generation_config.json +6 -0
- checkpoint-720/merges.txt +0 -0
- checkpoint-720/model.safetensors +3 -0
- checkpoint-720/optimizer.pt +3 -0
- checkpoint-720/rng_state.pth +3 -0
- checkpoint-720/scheduler.pt +3 -0
- checkpoint-720/special_tokens_map.json +20 -0
- checkpoint-720/tokenizer.json +0 -0
- checkpoint-720/tokenizer_config.json +43 -0
- checkpoint-720/trainer_state.json +28 -0
- checkpoint-720/training_args.bin +3 -0
- checkpoint-720/vocab.json +0 -0
- config.json +28 -0
- generation_config.json +6 -0
- merges.txt +0 -0
- model.safetensors +3 -0
- special_tokens_map.json +20 -0
- tokenizer.json +0 -0
- tokenizer_config.json +43 -0
- train_results.json +9 -0
- trainer_state.json +45 -0
- training_args.bin +3 -0
- vocab.json +0 -0
README.md
ADDED
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
license: apache-2.0
|
3 |
+
base_model: sail/Sailor-0.5B
|
4 |
+
tags:
|
5 |
+
- trl
|
6 |
+
- sft
|
7 |
+
- generated_from_trainer
|
8 |
+
model-index:
|
9 |
+
- name: pirate-0.5
|
10 |
+
results: []
|
11 |
+
---
|
12 |
+
|
13 |
+
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
|
14 |
+
should probably proofread and complete it, then remove this comment. -->
|
15 |
+
|
16 |
+
# pirate-0.5
|
17 |
+
|
18 |
+
This model is a fine-tuned version of [sail/Sailor-0.5B](https://huggingface.co/sail/Sailor-0.5B) on an unknown dataset.
|
19 |
+
|
20 |
+
## Model description
|
21 |
+
|
22 |
+
More information needed
|
23 |
+
|
24 |
+
## Intended uses & limitations
|
25 |
+
|
26 |
+
More information needed
|
27 |
+
|
28 |
+
## Training and evaluation data
|
29 |
+
|
30 |
+
More information needed
|
31 |
+
|
32 |
+
## Training procedure
|
33 |
+
|
34 |
+
### Training hyperparameters
|
35 |
+
|
36 |
+
The following hyperparameters were used during training:
|
37 |
+
- learning_rate: 0.0001
|
38 |
+
- train_batch_size: 1
|
39 |
+
- eval_batch_size: 1
|
40 |
+
- seed: 42
|
41 |
+
- gradient_accumulation_steps: 16
|
42 |
+
- total_train_batch_size: 16
|
43 |
+
- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
|
44 |
+
- lr_scheduler_type: linear
|
45 |
+
- num_epochs: 1
|
46 |
+
|
47 |
+
### Framework versions
|
48 |
+
|
49 |
+
- Transformers 4.39.0
|
50 |
+
- Pytorch 2.2.1+cu121
|
51 |
+
- Datasets 2.18.0
|
52 |
+
- Tokenizers 0.15.2
|
added_tokens.json
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"<|endoftext|>": 151643,
|
3 |
+
"<|im_end|>": 151645,
|
4 |
+
"<|im_start|>": 151644
|
5 |
+
}
|
all_results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"epoch": 1.0,
|
3 |
+
"total_flos": 6.586350857040691e+16,
|
4 |
+
"train_loss": 2.345679919805553,
|
5 |
+
"train_runtime": 10097.1443,
|
6 |
+
"train_samples": 11564,
|
7 |
+
"train_samples_per_second": 1.145,
|
8 |
+
"train_steps_per_second": 0.072
|
9 |
+
}
|
checkpoint-704/added_tokens.json
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"<|endoftext|>": 151643,
|
3 |
+
"<|im_end|>": 151645,
|
4 |
+
"<|im_start|>": 151644
|
5 |
+
}
|
checkpoint-704/config.json
ADDED
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_name_or_path": "sail/Sailor-0.5B",
|
3 |
+
"architectures": [
|
4 |
+
"Qwen2ForCausalLM"
|
5 |
+
],
|
6 |
+
"attention_dropout": 0.0,
|
7 |
+
"bos_token_id": 151643,
|
8 |
+
"eos_token_id": 151643,
|
9 |
+
"hidden_act": "silu",
|
10 |
+
"hidden_size": 1024,
|
11 |
+
"initializer_range": 0.02,
|
12 |
+
"intermediate_size": 2816,
|
13 |
+
"max_position_embeddings": 32768,
|
14 |
+
"max_window_layers": 21,
|
15 |
+
"model_type": "qwen2",
|
16 |
+
"num_attention_heads": 16,
|
17 |
+
"num_hidden_layers": 24,
|
18 |
+
"num_key_value_heads": 16,
|
19 |
+
"rms_norm_eps": 1e-06,
|
20 |
+
"rope_theta": 1000000.0,
|
21 |
+
"sliding_window": 32768,
|
22 |
+
"tie_word_embeddings": false,
|
23 |
+
"torch_dtype": "float32",
|
24 |
+
"transformers_version": "4.39.0",
|
25 |
+
"use_cache": false,
|
26 |
+
"use_sliding_window": false,
|
27 |
+
"vocab_size": 151936
|
28 |
+
}
|
checkpoint-704/generation_config.json
ADDED
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"bos_token_id": 151643,
|
3 |
+
"eos_token_id": 151643,
|
4 |
+
"max_new_tokens": 2048,
|
5 |
+
"transformers_version": "4.39.0"
|
6 |
+
}
|
checkpoint-704/merges.txt
ADDED
The diff for this file is too large to render.
See raw diff
|
|
checkpoint-704/model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:4ac8505106ce5bce1ae65a13be27e7c99cc3113e26b6da42fd7316dc80546fb8
|
3 |
+
size 2478313760
|
checkpoint-704/optimizer.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:57a973a5b8394aeda9a3416f629a91b730807b4450b13db60cbd8b263418c24f
|
3 |
+
size 4956808758
|
checkpoint-704/rng_state.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:2836f8619966597a96aa93766bae75f701ea0f05f6c5ec4da2400e566eb2157e
|
3 |
+
size 14180
|
checkpoint-704/scheduler.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:25593381fb2d18a57982e3685b21e92e41bb3756b66d314972de8531fe86d4d9
|
3 |
+
size 1064
|
checkpoint-704/special_tokens_map.json
ADDED
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"additional_special_tokens": [
|
3 |
+
"<|im_start|>",
|
4 |
+
"<|im_end|>"
|
5 |
+
],
|
6 |
+
"eos_token": {
|
7 |
+
"content": "<|endoftext|>",
|
8 |
+
"lstrip": false,
|
9 |
+
"normalized": false,
|
10 |
+
"rstrip": false,
|
11 |
+
"single_word": false
|
12 |
+
},
|
13 |
+
"pad_token": {
|
14 |
+
"content": "<|endoftext|>",
|
15 |
+
"lstrip": false,
|
16 |
+
"normalized": false,
|
17 |
+
"rstrip": false,
|
18 |
+
"single_word": false
|
19 |
+
}
|
20 |
+
}
|
checkpoint-704/tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
checkpoint-704/tokenizer_config.json
ADDED
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"add_prefix_space": false,
|
3 |
+
"added_tokens_decoder": {
|
4 |
+
"151643": {
|
5 |
+
"content": "<|endoftext|>",
|
6 |
+
"lstrip": false,
|
7 |
+
"normalized": false,
|
8 |
+
"rstrip": false,
|
9 |
+
"single_word": false,
|
10 |
+
"special": true
|
11 |
+
},
|
12 |
+
"151644": {
|
13 |
+
"content": "<|im_start|>",
|
14 |
+
"lstrip": false,
|
15 |
+
"normalized": false,
|
16 |
+
"rstrip": false,
|
17 |
+
"single_word": false,
|
18 |
+
"special": true
|
19 |
+
},
|
20 |
+
"151645": {
|
21 |
+
"content": "<|im_end|>",
|
22 |
+
"lstrip": false,
|
23 |
+
"normalized": false,
|
24 |
+
"rstrip": false,
|
25 |
+
"single_word": false,
|
26 |
+
"special": true
|
27 |
+
}
|
28 |
+
},
|
29 |
+
"additional_special_tokens": [
|
30 |
+
"<|im_start|>",
|
31 |
+
"<|im_end|>"
|
32 |
+
],
|
33 |
+
"bos_token": null,
|
34 |
+
"chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
|
35 |
+
"clean_up_tokenization_spaces": false,
|
36 |
+
"eos_token": "<|endoftext|>",
|
37 |
+
"errors": "replace",
|
38 |
+
"model_max_length": 2048,
|
39 |
+
"pad_token": "<|endoftext|>",
|
40 |
+
"split_special_tokens": false,
|
41 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
42 |
+
"unk_token": null
|
43 |
+
}
|
checkpoint-704/trainer_state.json
ADDED
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"best_metric": null,
|
3 |
+
"best_model_checkpoint": null,
|
4 |
+
"epoch": 0.9740574195780007,
|
5 |
+
"eval_steps": 500,
|
6 |
+
"global_step": 704,
|
7 |
+
"is_hyper_param_search": false,
|
8 |
+
"is_local_process_zero": true,
|
9 |
+
"is_world_process_zero": true,
|
10 |
+
"log_history": [
|
11 |
+
{
|
12 |
+
"epoch": 0.69,
|
13 |
+
"grad_norm": 1.6492760181427002,
|
14 |
+
"learning_rate": 3.074792243767313e-05,
|
15 |
+
"loss": 2.393,
|
16 |
+
"step": 500
|
17 |
+
}
|
18 |
+
],
|
19 |
+
"logging_steps": 500,
|
20 |
+
"max_steps": 722,
|
21 |
+
"num_input_tokens_seen": 0,
|
22 |
+
"num_train_epochs": 1,
|
23 |
+
"save_steps": 16,
|
24 |
+
"total_flos": 6.422148204095078e+16,
|
25 |
+
"train_batch_size": 1,
|
26 |
+
"trial_name": null,
|
27 |
+
"trial_params": null
|
28 |
+
}
|
checkpoint-704/training_args.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b7d4ae8d399268e950a2907d698c780c677d3e9ca2ca197ca7574643ca3afddb
|
3 |
+
size 4920
|
checkpoint-704/vocab.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
checkpoint-720/added_tokens.json
ADDED
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"<|endoftext|>": 151643,
|
3 |
+
"<|im_end|>": 151645,
|
4 |
+
"<|im_start|>": 151644
|
5 |
+
}
|
checkpoint-720/config.json
ADDED
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_name_or_path": "sail/Sailor-0.5B",
|
3 |
+
"architectures": [
|
4 |
+
"Qwen2ForCausalLM"
|
5 |
+
],
|
6 |
+
"attention_dropout": 0.0,
|
7 |
+
"bos_token_id": 151643,
|
8 |
+
"eos_token_id": 151643,
|
9 |
+
"hidden_act": "silu",
|
10 |
+
"hidden_size": 1024,
|
11 |
+
"initializer_range": 0.02,
|
12 |
+
"intermediate_size": 2816,
|
13 |
+
"max_position_embeddings": 32768,
|
14 |
+
"max_window_layers": 21,
|
15 |
+
"model_type": "qwen2",
|
16 |
+
"num_attention_heads": 16,
|
17 |
+
"num_hidden_layers": 24,
|
18 |
+
"num_key_value_heads": 16,
|
19 |
+
"rms_norm_eps": 1e-06,
|
20 |
+
"rope_theta": 1000000.0,
|
21 |
+
"sliding_window": 32768,
|
22 |
+
"tie_word_embeddings": false,
|
23 |
+
"torch_dtype": "float32",
|
24 |
+
"transformers_version": "4.39.0",
|
25 |
+
"use_cache": false,
|
26 |
+
"use_sliding_window": false,
|
27 |
+
"vocab_size": 151936
|
28 |
+
}
|
checkpoint-720/generation_config.json
ADDED
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"bos_token_id": 151643,
|
3 |
+
"eos_token_id": 151643,
|
4 |
+
"max_new_tokens": 2048,
|
5 |
+
"transformers_version": "4.39.0"
|
6 |
+
}
|
checkpoint-720/merges.txt
ADDED
The diff for this file is too large to render.
See raw diff
|
|
checkpoint-720/model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:6212ecbd55c15204cf42b1cd0ac1b84ad584fbe6d4244e51ed4d98c23436cdaa
|
3 |
+
size 2478313760
|
checkpoint-720/optimizer.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:7e0d2a6a7ddf48e96eaab0d3002b43c61a5bf9d954988cff21babbaa9e093334
|
3 |
+
size 4956808758
|
checkpoint-720/rng_state.pth
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:2836f8619966597a96aa93766bae75f701ea0f05f6c5ec4da2400e566eb2157e
|
3 |
+
size 14180
|
checkpoint-720/scheduler.pt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:e143127b26e54908981dd0613514cd8815de95036afbec20c87b2202ee37a429
|
3 |
+
size 1064
|
checkpoint-720/special_tokens_map.json
ADDED
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"additional_special_tokens": [
|
3 |
+
"<|im_start|>",
|
4 |
+
"<|im_end|>"
|
5 |
+
],
|
6 |
+
"eos_token": {
|
7 |
+
"content": "<|endoftext|>",
|
8 |
+
"lstrip": false,
|
9 |
+
"normalized": false,
|
10 |
+
"rstrip": false,
|
11 |
+
"single_word": false
|
12 |
+
},
|
13 |
+
"pad_token": {
|
14 |
+
"content": "<|endoftext|>",
|
15 |
+
"lstrip": false,
|
16 |
+
"normalized": false,
|
17 |
+
"rstrip": false,
|
18 |
+
"single_word": false
|
19 |
+
}
|
20 |
+
}
|
checkpoint-720/tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
checkpoint-720/tokenizer_config.json
ADDED
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"add_prefix_space": false,
|
3 |
+
"added_tokens_decoder": {
|
4 |
+
"151643": {
|
5 |
+
"content": "<|endoftext|>",
|
6 |
+
"lstrip": false,
|
7 |
+
"normalized": false,
|
8 |
+
"rstrip": false,
|
9 |
+
"single_word": false,
|
10 |
+
"special": true
|
11 |
+
},
|
12 |
+
"151644": {
|
13 |
+
"content": "<|im_start|>",
|
14 |
+
"lstrip": false,
|
15 |
+
"normalized": false,
|
16 |
+
"rstrip": false,
|
17 |
+
"single_word": false,
|
18 |
+
"special": true
|
19 |
+
},
|
20 |
+
"151645": {
|
21 |
+
"content": "<|im_end|>",
|
22 |
+
"lstrip": false,
|
23 |
+
"normalized": false,
|
24 |
+
"rstrip": false,
|
25 |
+
"single_word": false,
|
26 |
+
"special": true
|
27 |
+
}
|
28 |
+
},
|
29 |
+
"additional_special_tokens": [
|
30 |
+
"<|im_start|>",
|
31 |
+
"<|im_end|>"
|
32 |
+
],
|
33 |
+
"bos_token": null,
|
34 |
+
"chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
|
35 |
+
"clean_up_tokenization_spaces": false,
|
36 |
+
"eos_token": "<|endoftext|>",
|
37 |
+
"errors": "replace",
|
38 |
+
"model_max_length": 2048,
|
39 |
+
"pad_token": "<|endoftext|>",
|
40 |
+
"split_special_tokens": false,
|
41 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
42 |
+
"unk_token": null
|
43 |
+
}
|
checkpoint-720/trainer_state.json
ADDED
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"best_metric": null,
|
3 |
+
"best_model_checkpoint": null,
|
4 |
+
"epoch": 0.9961950882047734,
|
5 |
+
"eval_steps": 500,
|
6 |
+
"global_step": 720,
|
7 |
+
"is_hyper_param_search": false,
|
8 |
+
"is_local_process_zero": true,
|
9 |
+
"is_world_process_zero": true,
|
10 |
+
"log_history": [
|
11 |
+
{
|
12 |
+
"epoch": 0.69,
|
13 |
+
"grad_norm": 1.6492760181427002,
|
14 |
+
"learning_rate": 3.074792243767313e-05,
|
15 |
+
"loss": 2.393,
|
16 |
+
"step": 500
|
17 |
+
}
|
18 |
+
],
|
19 |
+
"logging_steps": 500,
|
20 |
+
"max_steps": 722,
|
21 |
+
"num_input_tokens_seen": 0,
|
22 |
+
"num_train_epochs": 1,
|
23 |
+
"save_steps": 16,
|
24 |
+
"total_flos": 6.568106117824512e+16,
|
25 |
+
"train_batch_size": 1,
|
26 |
+
"trial_name": null,
|
27 |
+
"trial_params": null
|
28 |
+
}
|
checkpoint-720/training_args.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b7d4ae8d399268e950a2907d698c780c677d3e9ca2ca197ca7574643ca3afddb
|
3 |
+
size 4920
|
checkpoint-720/vocab.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
config.json
ADDED
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_name_or_path": "sail/Sailor-0.5B",
|
3 |
+
"architectures": [
|
4 |
+
"Qwen2ForCausalLM"
|
5 |
+
],
|
6 |
+
"attention_dropout": 0.0,
|
7 |
+
"bos_token_id": 151643,
|
8 |
+
"eos_token_id": 151643,
|
9 |
+
"hidden_act": "silu",
|
10 |
+
"hidden_size": 1024,
|
11 |
+
"initializer_range": 0.02,
|
12 |
+
"intermediate_size": 2816,
|
13 |
+
"max_position_embeddings": 32768,
|
14 |
+
"max_window_layers": 21,
|
15 |
+
"model_type": "qwen2",
|
16 |
+
"num_attention_heads": 16,
|
17 |
+
"num_hidden_layers": 24,
|
18 |
+
"num_key_value_heads": 16,
|
19 |
+
"rms_norm_eps": 1e-06,
|
20 |
+
"rope_theta": 1000000.0,
|
21 |
+
"sliding_window": 32768,
|
22 |
+
"tie_word_embeddings": false,
|
23 |
+
"torch_dtype": "float32",
|
24 |
+
"transformers_version": "4.39.0",
|
25 |
+
"use_cache": false,
|
26 |
+
"use_sliding_window": false,
|
27 |
+
"vocab_size": 151936
|
28 |
+
}
|
generation_config.json
ADDED
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"bos_token_id": 151643,
|
3 |
+
"eos_token_id": 151643,
|
4 |
+
"max_new_tokens": 2048,
|
5 |
+
"transformers_version": "4.39.0"
|
6 |
+
}
|
merges.txt
ADDED
The diff for this file is too large to render.
See raw diff
|
|
model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:463fd004ca4166b871e4ecac22a31dada7611b529e6000fd11ba7ccd7cf5c92b
|
3 |
+
size 2478313760
|
special_tokens_map.json
ADDED
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"additional_special_tokens": [
|
3 |
+
"<|im_start|>",
|
4 |
+
"<|im_end|>"
|
5 |
+
],
|
6 |
+
"eos_token": {
|
7 |
+
"content": "<|endoftext|>",
|
8 |
+
"lstrip": false,
|
9 |
+
"normalized": false,
|
10 |
+
"rstrip": false,
|
11 |
+
"single_word": false
|
12 |
+
},
|
13 |
+
"pad_token": {
|
14 |
+
"content": "<|endoftext|>",
|
15 |
+
"lstrip": false,
|
16 |
+
"normalized": false,
|
17 |
+
"rstrip": false,
|
18 |
+
"single_word": false
|
19 |
+
}
|
20 |
+
}
|
tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
tokenizer_config.json
ADDED
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"add_prefix_space": false,
|
3 |
+
"added_tokens_decoder": {
|
4 |
+
"151643": {
|
5 |
+
"content": "<|endoftext|>",
|
6 |
+
"lstrip": false,
|
7 |
+
"normalized": false,
|
8 |
+
"rstrip": false,
|
9 |
+
"single_word": false,
|
10 |
+
"special": true
|
11 |
+
},
|
12 |
+
"151644": {
|
13 |
+
"content": "<|im_start|>",
|
14 |
+
"lstrip": false,
|
15 |
+
"normalized": false,
|
16 |
+
"rstrip": false,
|
17 |
+
"single_word": false,
|
18 |
+
"special": true
|
19 |
+
},
|
20 |
+
"151645": {
|
21 |
+
"content": "<|im_end|>",
|
22 |
+
"lstrip": false,
|
23 |
+
"normalized": false,
|
24 |
+
"rstrip": false,
|
25 |
+
"single_word": false,
|
26 |
+
"special": true
|
27 |
+
}
|
28 |
+
},
|
29 |
+
"additional_special_tokens": [
|
30 |
+
"<|im_start|>",
|
31 |
+
"<|im_end|>"
|
32 |
+
],
|
33 |
+
"bos_token": null,
|
34 |
+
"chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
|
35 |
+
"clean_up_tokenization_spaces": false,
|
36 |
+
"eos_token": "<|endoftext|>",
|
37 |
+
"errors": "replace",
|
38 |
+
"model_max_length": 2048,
|
39 |
+
"pad_token": "<|endoftext|>",
|
40 |
+
"split_special_tokens": false,
|
41 |
+
"tokenizer_class": "Qwen2Tokenizer",
|
42 |
+
"unk_token": null
|
43 |
+
}
|
train_results.json
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"epoch": 1.0,
|
3 |
+
"total_flos": 6.586350857040691e+16,
|
4 |
+
"train_loss": 2.345679919805553,
|
5 |
+
"train_runtime": 10097.1443,
|
6 |
+
"train_samples": 11564,
|
7 |
+
"train_samples_per_second": 1.145,
|
8 |
+
"train_steps_per_second": 0.072
|
9 |
+
}
|
trainer_state.json
ADDED
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"best_metric": null,
|
3 |
+
"best_model_checkpoint": null,
|
4 |
+
"epoch": 0.99896229678312,
|
5 |
+
"eval_steps": 500,
|
6 |
+
"global_step": 722,
|
7 |
+
"is_hyper_param_search": false,
|
8 |
+
"is_local_process_zero": true,
|
9 |
+
"is_world_process_zero": true,
|
10 |
+
"log_history": [
|
11 |
+
{
|
12 |
+
"epoch": 0.69,
|
13 |
+
"grad_norm": 1.6492760181427002,
|
14 |
+
"learning_rate": 3.074792243767313e-05,
|
15 |
+
"loss": 2.393,
|
16 |
+
"step": 500
|
17 |
+
},
|
18 |
+
{
|
19 |
+
"epoch": 1.0,
|
20 |
+
"eval_loss": 2.211937189102173,
|
21 |
+
"eval_runtime": 268.8732,
|
22 |
+
"eval_samples_per_second": 4.779,
|
23 |
+
"eval_steps_per_second": 4.779,
|
24 |
+
"step": 722
|
25 |
+
},
|
26 |
+
{
|
27 |
+
"epoch": 1.0,
|
28 |
+
"step": 722,
|
29 |
+
"total_flos": 6.586350857040691e+16,
|
30 |
+
"train_loss": 2.345679919805553,
|
31 |
+
"train_runtime": 10097.1443,
|
32 |
+
"train_samples_per_second": 1.145,
|
33 |
+
"train_steps_per_second": 0.072
|
34 |
+
}
|
35 |
+
],
|
36 |
+
"logging_steps": 500,
|
37 |
+
"max_steps": 722,
|
38 |
+
"num_input_tokens_seen": 0,
|
39 |
+
"num_train_epochs": 1,
|
40 |
+
"save_steps": 16,
|
41 |
+
"total_flos": 6.586350857040691e+16,
|
42 |
+
"train_batch_size": 1,
|
43 |
+
"trial_name": null,
|
44 |
+
"trial_params": null
|
45 |
+
}
|
training_args.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:e53c5206754d6bc986a8887120b97ccfab44b73a311041049e80ef1642ed8e8d
|
3 |
+
size 4920
|
vocab.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|