Hugging Face's logo

bs-la
/
bloom-560m_az_bitfit_10000samples_-1vocab_original-frozen

Text Generation
Transformers
PyTorch
bloom
text-generation-inference
Inference Endpoints
Model card Files Community
yongzx commited on
Commit
c1109e2
·
1 Parent(s): 7aad5a4

rm --cached *

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +0 -41
  2. .gitignore +0 -23
  3. all_results.json +0 -14
  4. checkpoint-10000/config.json +0 -39
  5. checkpoint-10000/optimizer.pt +0 -3
  6. checkpoint-10000/pytorch_model.bin +0 -3
  7. checkpoint-10000/rng_state.pth +0 -3
  8. checkpoint-10000/scheduler.pt +0 -3
  9. checkpoint-10000/special_tokens_map.json +0 -1
  10. checkpoint-10000/tokenizer.json +0 -3
  11. checkpoint-10000/tokenizer_config.json +0 -1
  12. checkpoint-10000/trainer_state.json +0 -56
  13. checkpoint-10000/training_args.bin +0 -3
  14. checkpoint-10000/wikiann-az-results.txt +0 -8
  15. checkpoint-12500/config.json +0 -39
  16. checkpoint-12500/optimizer.pt +0 -3
  17. checkpoint-12500/pytorch_model.bin +0 -3
  18. checkpoint-12500/rng_state.pth +0 -3
  19. checkpoint-12500/scheduler.pt +0 -3
  20. checkpoint-12500/special_tokens_map.json +0 -1
  21. checkpoint-12500/tokenizer.json +0 -3
  22. checkpoint-12500/tokenizer_config.json +0 -1
  23. checkpoint-12500/trainer_state.json +0 -86
  24. checkpoint-12500/training_args.bin +0 -3
  25. checkpoint-12500/wikiann-az-results.txt +0 -8
  26. checkpoint-15000/config.json +0 -39
  27. checkpoint-15000/optimizer.pt +0 -3
  28. checkpoint-15000/pytorch_model.bin +0 -3
  29. checkpoint-15000/rng_state.pth +0 -3
  30. checkpoint-15000/scheduler.pt +0 -3
  31. checkpoint-15000/special_tokens_map.json +0 -1
  32. checkpoint-15000/tokenizer.json +0 -3
  33. checkpoint-15000/tokenizer_config.json +0 -1
  34. checkpoint-15000/trainer_state.json +0 -76
  35. checkpoint-15000/training_args.bin +0 -3
  36. checkpoint-20000/config.json +0 -39
  37. checkpoint-20000/optimizer.pt +0 -3
  38. checkpoint-20000/pytorch_model.bin +0 -3
  39. checkpoint-20000/rng_state.pth +0 -3
  40. checkpoint-20000/scheduler.pt +0 -3
  41. checkpoint-20000/special_tokens_map.json +0 -1
  42. checkpoint-20000/tokenizer.json +0 -3
  43. checkpoint-20000/tokenizer_config.json +0 -1
  44. checkpoint-20000/trainer_state.json +0 -96
  45. checkpoint-20000/training_args.bin +0 -3
  46. checkpoint-2500/config.json +0 -39
  47. checkpoint-2500/optimizer.pt +0 -3
  48. checkpoint-2500/pytorch_model.bin +0 -3
  49. checkpoint-2500/rng_state.pth +0 -3
  50. checkpoint-2500/scheduler.pt +0 -3
.gitattributes DELETED
@@ -1,41 +0,0 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ftz filter=lfs diff=lfs merge=lfs -text
6
- *.gz filter=lfs diff=lfs merge=lfs -text
7
- *.h5 filter=lfs diff=lfs merge=lfs -text
8
- *.joblib filter=lfs diff=lfs merge=lfs -text
9
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
10
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
11
- *.model filter=lfs diff=lfs merge=lfs -text
12
- *.msgpack filter=lfs diff=lfs merge=lfs -text
13
- *.npy filter=lfs diff=lfs merge=lfs -text
14
- *.npz filter=lfs diff=lfs merge=lfs -text
15
- *.onnx filter=lfs diff=lfs merge=lfs -text
16
- *.ot filter=lfs diff=lfs merge=lfs -text
17
- *.parquet filter=lfs diff=lfs merge=lfs -text
18
- *.pb filter=lfs diff=lfs merge=lfs -text
19
- *.pickle filter=lfs diff=lfs merge=lfs -text
20
- *.pkl filter=lfs diff=lfs merge=lfs -text
21
- *.pt filter=lfs diff=lfs merge=lfs -text
22
- *.pth filter=lfs diff=lfs merge=lfs -text
23
- *.rar filter=lfs diff=lfs merge=lfs -text
24
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
25
- *.tar.* filter=lfs diff=lfs merge=lfs -text
26
- *.tflite filter=lfs diff=lfs merge=lfs -text
27
- *.tgz filter=lfs diff=lfs merge=lfs -text
28
- *.wasm filter=lfs diff=lfs merge=lfs -text
29
- *.xz filter=lfs diff=lfs merge=lfs -text
30
- *.zip filter=lfs diff=lfs merge=lfs -text
31
- *.zst filter=lfs diff=lfs merge=lfs -text
32
- *tfevents* filter=lfs diff=lfs merge=lfs -text
33
- checkpoint-10000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
34
- checkpoint-12500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
35
- checkpoint-15000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
36
- checkpoint-20000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
37
- checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
38
- checkpoint-25000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
39
- checkpoint-5000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
40
- checkpoint-7500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
41
- tokenizer.json filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
.gitignore DELETED
@@ -1,23 +0,0 @@
1
- */pilot_*/
2
- pilot_*/
3
- checkpoint-*/
4
- */pilot_*/
5
- pilot_*/
6
- checkpoint-*/
7
- */pilot_*/
8
- pilot_*/
9
- checkpoint-*/
10
- */pilot_*/
11
- pilot_*/
12
- checkpoint-*/
13
- */pilot_*/
14
- pilot_*/
15
- checkpoint-*/
16
- */pilot_*/
17
- pilot_*/
18
- checkpoint-*/
19
- */pilot_*/
20
- pilot_*/
21
- checkpoint-*/
22
- */pilot_*/
23
- pilot_*/
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
all_results.json DELETED
@@ -1,14 +0,0 @@
1
- {
2
- "epoch": 19.86,
3
- "eval_loss": 3.520382881164551,
4
- "eval_runtime": 518.4337,
5
- "eval_samples": 4906,
6
- "eval_samples_per_second": 9.463,
7
- "eval_steps_per_second": 4.732,
8
- "perplexity": 33.79736636184415,
9
- "train_loss": 3.6973518359375,
10
- "train_runtime": 53135.1163,
11
- "train_samples": 10000,
12
- "train_samples_per_second": 3.764,
13
- "train_steps_per_second": 0.47
14
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-10000/config.json DELETED
@@ -1,39 +0,0 @@
1
- {
2
- "_name_or_path": "bigscience/bloom-350m",
3
- "adapters": {
4
- "adapters": {},
5
- "config_map": {},
6
- "fusion_config_map": {},
7
- "fusions": {}
8
- },
9
- "apply_residual_connection_post_layernorm": false,
10
- "architectures": [
11
- "BloomForCausalLM"
12
- ],
13
- "attention_dropout": 0.0,
14
- "attention_softmax_in_fp32": true,
15
- "bias_dropout_fusion": true,
16
- "bos_token_id": 1,
17
- "eos_token_id": 2,
18
- "hidden_dropout": 0.0,
19
- "hidden_size": 1024,
20
- "initializer_range": 0.02,
21
- "layer_norm_epsilon": 1e-05,
22
- "masked_softmax_fusion": true,
23
- "model_type": "bloom",
24
- "n_head": 16,
25
- "n_inner": null,
26
- "n_layer": 24,
27
- "offset_alibi": 100,
28
- "pad_token_id": 3,
29
- "pretraining_tp": 1,
30
- "seq_length": 2048,
31
- "skip_bias_add": true,
32
- "skip_bias_add_qkv": false,
33
- "slow_but_exact": false,
34
- "torch_dtype": "float32",
35
- "transformers_version": "4.20.0.dev0",
36
- "unk_token_id": 0,
37
- "use_cache": true,
38
- "vocab_size": 250880
39
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-10000/optimizer.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:cf7f64bb919c1b75c964f253c50125ccf1e1684c0c9979785ab5f879599cba19
3
- size 2254269
 
 
 
 
checkpoint-10000/pytorch_model.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:fe3c7b0341df858cc1beabbbceb6eed9e315110b3e54ad2cb52d600ec787b536
3
- size 2236955191
 
 
 
 
checkpoint-10000/rng_state.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:ff0b7d688f72cecc3d6db8a5558dfafab2669c75076ea7f5e8d0318cba901963
3
- size 14503
 
 
 
 
checkpoint-10000/scheduler.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:ebae5cf74f470a9dc57b090feb9de29d57aa2d381061d1a61fd32b3c3221556b
3
- size 623
 
 
 
 
checkpoint-10000/special_tokens_map.json DELETED
@@ -1 +0,0 @@
1
- {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}
 
 
checkpoint-10000/tokenizer.json DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:8f6efc66e73f1fd69da4f436e48befb519fdff3fe18910850c1d41bd862293a5
3
- size 14500443
 
 
 
 
checkpoint-10000/tokenizer_config.json DELETED
@@ -1 +0,0 @@
1
- {"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "add_prefix_space": false, "name_or_path": "bigscience/bloom-350m", "special_tokens_map_file": null, "padding_side": "left", "tokenizer_class": "BloomTokenizer"}
 
 
checkpoint-10000/trainer_state.json DELETED
@@ -1,56 +0,0 @@
1
- {
2
- "best_metric": 3.64797043800354,
3
- "best_model_checkpoint": "/users/zyong2/data/zyong2/bigscience/data/processed/024/bloom-350m_az_bitfit_10000samples_-1vocab_original-frozen/checkpoint-10000",
4
- "epoch": 7.942250446517166,
5
- "global_step": 10000,
6
- "is_hyper_param_search": false,
7
- "is_local_process_zero": true,
8
- "is_world_process_zero": true,
9
- "log_history": [
10
- {
11
- "epoch": 1.99,
12
- "learning_rate": 9e-05,
13
- "loss": 4.249,
14
- "step": 2500
15
- },
16
- {
17
- "epoch": 3.97,
18
- "learning_rate": 8e-05,
19
- "loss": 3.9172,
20
- "step": 5000
21
- },
22
- {
23
- "epoch": 3.97,
24
- "eval_loss": 3.8250739574432373,
25
- "eval_runtime": 519.4242,
26
- "eval_samples_per_second": 9.445,
27
- "eval_steps_per_second": 4.723,
28
- "step": 5000
29
- },
30
- {
31
- "epoch": 5.96,
32
- "learning_rate": 7e-05,
33
- "loss": 3.7702,
34
- "step": 7500
35
- },
36
- {
37
- "epoch": 7.94,
38
- "learning_rate": 6e-05,
39
- "loss": 3.6827,
40
- "step": 10000
41
- },
42
- {
43
- "epoch": 7.94,
44
- "eval_loss": 3.64797043800354,
45
- "eval_runtime": 519.0865,
46
- "eval_samples_per_second": 9.451,
47
- "eval_steps_per_second": 4.726,
48
- "step": 10000
49
- }
50
- ],
51
- "max_steps": 25000,
52
- "num_train_epochs": 20,
53
- "total_flos": 1.4865812724645888e+17,
54
- "trial_name": null,
55
- "trial_params": null
56
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-10000/training_args.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:0cd06de07bc00b172f616a37bcac81b46cefe15d7481c457bed5477be8eb5945
3
- size 3375
 
 
 
 
checkpoint-10000/wikiann-az-results.txt DELETED
@@ -1,8 +0,0 @@
1
- ==================================================
2
- Results
3
- ==================================================
4
- Model: /users/zyong2/data/zyong2/bigscience/data/processed/024/bloom-350m_az_bitfit_10000samples_-1vocab_original-frozen/checkpoint-10000
5
- [0.3831710709318498, 0.36646621860629647, 0.37618879887284257, 0.36916695471828553, 0.39875173370319006, 0.3918592618144188, 0.3883292810003473, 0.3762723762723763, 0.37241379310344824, 0.37391001046389954]
6
- 37.97
7
- 1.00
8
- ==================================================
 
 
 
 
 
 
 
 
 
checkpoint-12500/config.json DELETED
@@ -1,39 +0,0 @@
1
- {
2
- "_name_or_path": "bigscience/bloom-350m",
3
- "adapters": {
4
- "adapters": {},
5
- "config_map": {},
6
- "fusion_config_map": {},
7
- "fusions": {}
8
- },
9
- "apply_residual_connection_post_layernorm": false,
10
- "architectures": [
11
- "BloomForCausalLM"
12
- ],
13
- "attention_dropout": 0.0,
14
- "attention_softmax_in_fp32": true,
15
- "bias_dropout_fusion": true,
16
- "bos_token_id": 1,
17
- "eos_token_id": 2,
18
- "hidden_dropout": 0.0,
19
- "hidden_size": 1024,
20
- "initializer_range": 0.02,
21
- "layer_norm_epsilon": 1e-05,
22
- "masked_softmax_fusion": true,
23
- "model_type": "bloom",
24
- "n_head": 16,
25
- "n_inner": null,
26
- "n_layer": 24,
27
- "offset_alibi": 100,
28
- "pad_token_id": 3,
29
- "pretraining_tp": 1,
30
- "seq_length": 2048,
31
- "skip_bias_add": true,
32
- "skip_bias_add_qkv": false,
33
- "slow_but_exact": false,
34
- "torch_dtype": "float32",
35
- "transformers_version": "4.20.0.dev0",
36
- "unk_token_id": 0,
37
- "use_cache": true,
38
- "vocab_size": 250880
39
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-12500/optimizer.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:1ba0462ba8ad328e587ce96f04f4c5e6f69394ff3582a4828bf6d85660bfe06b
3
- size 2254269
 
 
 
 
checkpoint-12500/pytorch_model.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:ee64dfd5244f8dc74878e78f209466dc68d8ea2c528c3719121c12794dc9ec09
3
- size 2236955191
 
 
 
 
checkpoint-12500/rng_state.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:0217759d118441b4ad07c71fd1be8c09ef99c4c1b9ca5ace0ae209a8872fb667
3
- size 14503
 
 
 
 
checkpoint-12500/scheduler.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:d270c6e3000cbdb534f7db7e774ca17393c2523690c8058754d752dd5b11a93a
3
- size 623
 
 
 
 
checkpoint-12500/special_tokens_map.json DELETED
@@ -1 +0,0 @@
1
- {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}
 
 
checkpoint-12500/tokenizer.json DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:8f6efc66e73f1fd69da4f436e48befb519fdff3fe18910850c1d41bd862293a5
3
- size 14500443
 
 
 
 
checkpoint-12500/tokenizer_config.json DELETED
@@ -1 +0,0 @@
1
- {"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "add_prefix_space": false, "name_or_path": "bigscience/bloom-350m", "special_tokens_map_file": null, "padding_side": "left", "tokenizer_class": "BloomTokenizer"}
 
 
checkpoint-12500/trainer_state.json DELETED
@@ -1,86 +0,0 @@
1
- {
2
- "best_metric": 3.1985085010528564,
3
- "best_model_checkpoint": "/users/zyong2/data/zyong2/bigscience/data/processed/024/bloom-350m_az_bitfit_10000samples_-1vocab_original-frozen/checkpoint-12500",
4
- "epoch": 9.927961897201826,
5
- "global_step": 12500,
6
- "is_hyper_param_search": false,
7
- "is_local_process_zero": true,
8
- "is_world_process_zero": true,
9
- "log_history": [
10
- {
11
- "epoch": 1.99,
12
- "learning_rate": 0.0008,
13
- "loss": 3.6925,
14
- "step": 2500
15
- },
16
- {
17
- "epoch": 1.99,
18
- "eval_loss": 3.425896406173706,
19
- "eval_runtime": 510.13,
20
- "eval_samples_per_second": 9.617,
21
- "eval_steps_per_second": 4.809,
22
- "step": 2500
23
- },
24
- {
25
- "epoch": 3.97,
26
- "learning_rate": 0.0006,
27
- "loss": 3.3514,
28
- "step": 5000
29
- },
30
- {
31
- "epoch": 3.97,
32
- "eval_loss": 3.297409772872925,
33
- "eval_runtime": 509.9866,
34
- "eval_samples_per_second": 9.62,
35
- "eval_steps_per_second": 4.81,
36
- "step": 5000
37
- },
38
- {
39
- "epoch": 5.96,
40
- "learning_rate": 0.0004,
41
- "loss": 3.2632,
42
- "step": 7500
43
- },
44
- {
45
- "epoch": 5.96,
46
- "eval_loss": 3.2419724464416504,
47
- "eval_runtime": 509.6021,
48
- "eval_samples_per_second": 9.627,
49
- "eval_steps_per_second": 4.814,
50
- "step": 7500
51
- },
52
- {
53
- "epoch": 7.94,
54
- "learning_rate": 0.0002,
55
- "loss": 3.2189,
56
- "step": 10000
57
- },
58
- {
59
- "epoch": 7.94,
60
- "eval_loss": 3.210970163345337,
61
- "eval_runtime": 509.6579,
62
- "eval_samples_per_second": 9.626,
63
- "eval_steps_per_second": 4.813,
64
- "step": 10000
65
- },
66
- {
67
- "epoch": 9.93,
68
- "learning_rate": 0.0,
69
- "loss": 3.1953,
70
- "step": 12500
71
- },
72
- {
73
- "epoch": 9.93,
74
- "eval_loss": 3.1985085010528564,
75
- "eval_runtime": 509.537,
76
- "eval_samples_per_second": 9.628,
77
- "eval_steps_per_second": 4.814,
78
- "step": 12500
79
- }
80
- ],
81
- "max_steps": 12500,
82
- "num_train_epochs": 10,
83
- "total_flos": 1.8582498082553856e+17,
84
- "trial_name": null,
85
- "trial_params": null
86
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-12500/training_args.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:b5a0101fd8f6900559ae4c013c2312163c0f239c35b8f9e14dabf6d0adafea84
3
- size 3375
 
 
 
 
checkpoint-12500/wikiann-az-results.txt DELETED
@@ -1,8 +0,0 @@
1
- ==================================================
2
- Results
3
- ==================================================
4
- Model: /users/zyong2/data/zyong2/bigscience/data/processed/024/bloom-350m_az_bitfit_10000samples_-1vocab_original-frozen/checkpoint-12500
5
- [0.3931564245810055, 0.3662171753016323, 0.37123627346794197, 0.37279944770452195, 0.3948646773074254, 0.3877904960110995, 0.37873523280055593, 0.3800491745697225, 0.3798690106859704, 0.37486910994764405]
6
- 38.00
7
- 0.90
8
- ==================================================
 
 
 
 
 
 
 
 
 
checkpoint-15000/config.json DELETED
@@ -1,39 +0,0 @@
1
- {
2
- "_name_or_path": "bigscience/bloom-350m",
3
- "adapters": {
4
- "adapters": {},
5
- "config_map": {},
6
- "fusion_config_map": {},
7
- "fusions": {}
8
- },
9
- "apply_residual_connection_post_layernorm": false,
10
- "architectures": [
11
- "BloomForCausalLM"
12
- ],
13
- "attention_dropout": 0.0,
14
- "attention_softmax_in_fp32": true,
15
- "bias_dropout_fusion": true,
16
- "bos_token_id": 1,
17
- "eos_token_id": 2,
18
- "hidden_dropout": 0.0,
19
- "hidden_size": 1024,
20
- "initializer_range": 0.02,
21
- "layer_norm_epsilon": 1e-05,
22
- "masked_softmax_fusion": true,
23
- "model_type": "bloom",
24
- "n_head": 16,
25
- "n_inner": null,
26
- "n_layer": 24,
27
- "offset_alibi": 100,
28
- "pad_token_id": 3,
29
- "pretraining_tp": 1,
30
- "seq_length": 2048,
31
- "skip_bias_add": true,
32
- "skip_bias_add_qkv": false,
33
- "slow_but_exact": false,
34
- "torch_dtype": "float32",
35
- "transformers_version": "4.20.0.dev0",
36
- "unk_token_id": 0,
37
- "use_cache": true,
38
- "vocab_size": 250880
39
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-15000/optimizer.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:93338749e626564421d833643e2dd16d8515d99d53a300869d23436882e97cf2
3
- size 2254269
 
 
 
 
checkpoint-15000/pytorch_model.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:8e3f96f3abaf7a0f9af4115907c15df4c5565ed49672ae92cf29d31269fb6e7f
3
- size 2236955191
 
 
 
 
checkpoint-15000/rng_state.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:0217759d118441b4ad07c71fd1be8c09ef99c4c1b9ca5ace0ae209a8872fb667
3
- size 14503
 
 
 
 
checkpoint-15000/scheduler.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:301727affc0c0a4c1f25106f7fd12c059ede0526ba52733c25be949ad3bc04d7
3
- size 623
 
 
 
 
checkpoint-15000/special_tokens_map.json DELETED
@@ -1 +0,0 @@
1
- {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}
 
 
checkpoint-15000/tokenizer.json DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:8f6efc66e73f1fd69da4f436e48befb519fdff3fe18910850c1d41bd862293a5
3
- size 14500443
 
 
 
 
checkpoint-15000/tokenizer_config.json DELETED
@@ -1 +0,0 @@
1
- {"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "add_prefix_space": false, "name_or_path": "bigscience/bloom-350m", "special_tokens_map_file": null, "padding_side": "left", "tokenizer_class": "BloomTokenizer"}
 
 
checkpoint-15000/trainer_state.json DELETED
@@ -1,76 +0,0 @@
1
- {
2
- "best_metric": 3.5690791606903076,
3
- "best_model_checkpoint": "/users/zyong2/data/zyong2/bigscience/data/processed/024/bloom-350m_az_bitfit_10000samples_-1vocab_original-frozen/checkpoint-15000",
4
- "epoch": 11.913673347886485,
5
- "global_step": 15000,
6
- "is_hyper_param_search": false,
7
- "is_local_process_zero": true,
8
- "is_world_process_zero": true,
9
- "log_history": [
10
- {
11
- "epoch": 1.99,
12
- "learning_rate": 9e-05,
13
- "loss": 4.249,
14
- "step": 2500
15
- },
16
- {
17
- "epoch": 3.97,
18
- "learning_rate": 8e-05,
19
- "loss": 3.9172,
20
- "step": 5000
21
- },
22
- {
23
- "epoch": 3.97,
24
- "eval_loss": 3.8250739574432373,
25
- "eval_runtime": 519.4242,
26
- "eval_samples_per_second": 9.445,
27
- "eval_steps_per_second": 4.723,
28
- "step": 5000
29
- },
30
- {
31
- "epoch": 5.96,
32
- "learning_rate": 7e-05,
33
- "loss": 3.7702,
34
- "step": 7500
35
- },
36
- {
37
- "epoch": 7.94,
38
- "learning_rate": 6e-05,
39
- "loss": 3.6827,
40
- "step": 10000
41
- },
42
- {
43
- "epoch": 7.94,
44
- "eval_loss": 3.64797043800354,
45
- "eval_runtime": 519.0865,
46
- "eval_samples_per_second": 9.451,
47
- "eval_steps_per_second": 4.726,
48
- "step": 10000
49
- },
50
- {
51
- "epoch": 9.93,
52
- "learning_rate": 5e-05,
53
- "loss": 3.624,
54
- "step": 12500
55
- },
56
- {
57
- "epoch": 11.91,
58
- "learning_rate": 4e-05,
59
- "loss": 3.585,
60
- "step": 15000
61
- },
62
- {
63
- "epoch": 11.91,
64
- "eval_loss": 3.5690791606903076,
65
- "eval_runtime": 519.0226,
66
- "eval_samples_per_second": 9.452,
67
- "eval_steps_per_second": 4.726,
68
- "step": 15000
69
- }
70
- ],
71
- "max_steps": 25000,
72
- "num_train_epochs": 20,
73
- "total_flos": 2.2299183440461824e+17,
74
- "trial_name": null,
75
- "trial_params": null
76
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-15000/training_args.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:0cd06de07bc00b172f616a37bcac81b46cefe15d7481c457bed5477be8eb5945
3
- size 3375
 
 
 
 
checkpoint-20000/config.json DELETED
@@ -1,39 +0,0 @@
1
- {
2
- "_name_or_path": "bigscience/bloom-350m",
3
- "adapters": {
4
- "adapters": {},
5
- "config_map": {},
6
- "fusion_config_map": {},
7
- "fusions": {}
8
- },
9
- "apply_residual_connection_post_layernorm": false,
10
- "architectures": [
11
- "BloomForCausalLM"
12
- ],
13
- "attention_dropout": 0.0,
14
- "attention_softmax_in_fp32": true,
15
- "bias_dropout_fusion": true,
16
- "bos_token_id": 1,
17
- "eos_token_id": 2,
18
- "hidden_dropout": 0.0,
19
- "hidden_size": 1024,
20
- "initializer_range": 0.02,
21
- "layer_norm_epsilon": 1e-05,
22
- "masked_softmax_fusion": true,
23
- "model_type": "bloom",
24
- "n_head": 16,
25
- "n_inner": null,
26
- "n_layer": 24,
27
- "offset_alibi": 100,
28
- "pad_token_id": 3,
29
- "pretraining_tp": 1,
30
- "seq_length": 2048,
31
- "skip_bias_add": true,
32
- "skip_bias_add_qkv": false,
33
- "slow_but_exact": false,
34
- "torch_dtype": "float32",
35
- "transformers_version": "4.20.0.dev0",
36
- "unk_token_id": 0,
37
- "use_cache": true,
38
- "vocab_size": 250880
39
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-20000/optimizer.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:a002623dc1f3a7f26188c4a378e00394c1a949760f52844efbdd6f502ebdad5e
3
- size 2254269
 
 
 
 
checkpoint-20000/pytorch_model.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:70afd8810d9ba16c3568302d0d9d5f78dcba8edba783672ac9c11160a556f4f3
3
- size 2236955191
 
 
 
 
checkpoint-20000/rng_state.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:528887aeaf571c1dd9d1789c0fad11e336830c7f10d9174d25b3f236cf9a2aa4
3
- size 14503
 
 
 
 
checkpoint-20000/scheduler.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:69341a1831197b0345e31eaac56abf9ad4527cc56eba4b526818b4ffb6ef6dad
3
- size 623
 
 
 
 
checkpoint-20000/special_tokens_map.json DELETED
@@ -1 +0,0 @@
1
- {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}
 
 
checkpoint-20000/tokenizer.json DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:8f6efc66e73f1fd69da4f436e48befb519fdff3fe18910850c1d41bd862293a5
3
- size 14500443
 
 
 
 
checkpoint-20000/tokenizer_config.json DELETED
@@ -1 +0,0 @@
1
- {"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>", "add_prefix_space": false, "name_or_path": "bigscience/bloom-350m", "special_tokens_map_file": null, "padding_side": "left", "tokenizer_class": "BloomTokenizer"}
 
 
checkpoint-20000/trainer_state.json DELETED
@@ -1,96 +0,0 @@
1
- {
2
- "best_metric": 3.531822443008423,
3
- "best_model_checkpoint": "/users/zyong2/data/zyong2/bigscience/data/processed/024/bloom-350m_az_bitfit_10000samples_-1vocab_original-frozen/checkpoint-20000",
4
- "epoch": 15.885096249255804,
5
- "global_step": 20000,
6
- "is_hyper_param_search": false,
7
- "is_local_process_zero": true,
8
- "is_world_process_zero": true,
9
- "log_history": [
10
- {
11
- "epoch": 1.99,
12
- "learning_rate": 9e-05,
13
- "loss": 4.249,
14
- "step": 2500
15
- },
16
- {
17
- "epoch": 3.97,
18
- "learning_rate": 8e-05,
19
- "loss": 3.9172,
20
- "step": 5000
21
- },
22
- {
23
- "epoch": 3.97,
24
- "eval_loss": 3.8250739574432373,
25
- "eval_runtime": 519.4242,
26
- "eval_samples_per_second": 9.445,
27
- "eval_steps_per_second": 4.723,
28
- "step": 5000
29
- },
30
- {
31
- "epoch": 5.96,
32
- "learning_rate": 7e-05,
33
- "loss": 3.7702,
34
- "step": 7500
35
- },
36
- {
37
- "epoch": 7.94,
38
- "learning_rate": 6e-05,
39
- "loss": 3.6827,
40
- "step": 10000
41
- },
42
- {
43
- "epoch": 7.94,
44
- "eval_loss": 3.64797043800354,
45
- "eval_runtime": 519.0865,
46
- "eval_samples_per_second": 9.451,
47
- "eval_steps_per_second": 4.726,
48
- "step": 10000
49
- },
50
- {
51
- "epoch": 9.93,
52
- "learning_rate": 5e-05,
53
- "loss": 3.624,
54
- "step": 12500
55
- },
56
- {
57
- "epoch": 11.91,
58
- "learning_rate": 4e-05,
59
- "loss": 3.585,
60
- "step": 15000
61
- },
62
- {
63
- "epoch": 11.91,
64
- "eval_loss": 3.5690791606903076,
65
- "eval_runtime": 519.0226,
66
- "eval_samples_per_second": 9.452,
67
- "eval_steps_per_second": 4.726,
68
- "step": 15000
69
- },
70
- {
71
- "epoch": 13.9,
72
- "learning_rate": 3e-05,
73
- "loss": 3.5588,
74
- "step": 17500
75
- },
76
- {
77
- "epoch": 15.89,
78
- "learning_rate": 2e-05,
79
- "loss": 3.54,
80
- "step": 20000
81
- },
82
- {
83
- "epoch": 15.89,
84
- "eval_loss": 3.531822443008423,
85
- "eval_runtime": 518.4812,
86
- "eval_samples_per_second": 9.462,
87
- "eval_steps_per_second": 4.731,
88
- "step": 20000
89
- }
90
- ],
91
- "max_steps": 25000,
92
- "num_train_epochs": 20,
93
- "total_flos": 2.973255415627776e+17,
94
- "trial_name": null,
95
- "trial_params": null
96
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-20000/training_args.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:0cd06de07bc00b172f616a37bcac81b46cefe15d7481c457bed5477be8eb5945
3
- size 3375
 
 
 
 
checkpoint-2500/config.json DELETED
@@ -1,39 +0,0 @@
1
- {
2
- "_name_or_path": "bigscience/bloom-350m",
3
- "adapters": {
4
- "adapters": {},
5
- "config_map": {},
6
- "fusion_config_map": {},
7
- "fusions": {}
8
- },
9
- "apply_residual_connection_post_layernorm": false,
10
- "architectures": [
11
- "BloomForCausalLM"
12
- ],
13
- "attention_dropout": 0.0,
14
- "attention_softmax_in_fp32": true,
15
- "bias_dropout_fusion": true,
16
- "bos_token_id": 1,
17
- "eos_token_id": 2,
18
- "hidden_dropout": 0.0,
19
- "hidden_size": 1024,
20
- "initializer_range": 0.02,
21
- "layer_norm_epsilon": 1e-05,
22
- "masked_softmax_fusion": true,
23
- "model_type": "bloom",
24
- "n_head": 16,
25
- "n_inner": null,
26
- "n_layer": 24,
27
- "offset_alibi": 100,
28
- "pad_token_id": 3,
29
- "pretraining_tp": 1,
30
- "seq_length": 2048,
31
- "skip_bias_add": true,
32
- "skip_bias_add_qkv": false,
33
- "slow_but_exact": false,
34
- "torch_dtype": "float32",
35
- "transformers_version": "4.20.0.dev0",
36
- "unk_token_id": 0,
37
- "use_cache": true,
38
- "vocab_size": 250880
39
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
checkpoint-2500/optimizer.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:41760dbdcd7c2a0f19e33dbacd6b48ccb3c940c26ead21843a4bffcb5bff4393
3
- size 2254269
 
 
 
 
checkpoint-2500/pytorch_model.bin DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:dc2b7d0a79254f7841b16b221ab8c6c6bb887c251fc400c22639ffa8cd17539c
3
- size 2236955191
 
 
 
 
checkpoint-2500/rng_state.pth DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:674676e662eeb93778c2b153ffad13aa90b43355da1956ce0b1e01e72f48c8d7
3
- size 14503
 
 
 
 
checkpoint-2500/scheduler.pt DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:8848be77d5e16f8ad560a7262091b3d3fcd8d0f3fa50682054480c93bc684fe6
3
- size 623