contrain model
Browse files- scripts/TRAIN.md +4 -4
- scripts/contrain-model.yaml +1 -1
scripts/TRAIN.md
CHANGED
@@ -41,17 +41,17 @@ litgpt pretrain --config ./pretrain-model.yaml
|
|
41 |
```
|
42 |
|
43 |
```bash
|
44 |
-
litgpt convert_from_litgpt out/pretrain/final/ out/
|
45 |
cp config.json out/pretrain/final/
|
46 |
-
cp config.json out/
|
47 |
```
|
48 |
|
49 |
```python
|
50 |
import torch
|
51 |
from safetensors.torch import save_file
|
52 |
|
53 |
-
state_dict = torch.load('out/
|
54 |
-
save_file(state_dict, 'out/
|
55 |
```
|
56 |
|
57 |
### Continued Pretraining
|
|
|
41 |
```
|
42 |
|
43 |
```bash
|
44 |
+
litgpt convert_from_litgpt out/pretrain/final/ out/converted_pretrain
|
45 |
cp config.json out/pretrain/final/
|
46 |
+
cp config.json out/converted_pretrain/
|
47 |
```
|
48 |
|
49 |
```python
|
50 |
import torch
|
51 |
from safetensors.torch import save_file
|
52 |
|
53 |
+
state_dict = torch.load('out/converted_pretrain/model.pth', map_location='cpu')
|
54 |
+
save_file(state_dict, 'out/converted_pretrain/model.safetensors')
|
55 |
```
|
56 |
|
57 |
### Continued Pretraining
|
scripts/contrain-model.yaml
CHANGED
@@ -85,7 +85,7 @@ train:
|
|
85 |
max_steps:
|
86 |
|
87 |
# Limits the length of samples. Off by default (type: Optional[int], default: null)
|
88 |
-
max_seq_length:
|
89 |
|
90 |
# Whether to tie the embedding weights with the language modeling head weights. (type: Optional[bool], default: False)
|
91 |
tie_embeddings: true
|
|
|
85 |
max_steps:
|
86 |
|
87 |
# Limits the length of samples. Off by default (type: Optional[int], default: null)
|
88 |
+
max_seq_length:
|
89 |
|
90 |
# Whether to tie the embedding weights with the language modeling head weights. (type: Optional[bool], default: False)
|
91 |
tie_embeddings: true
|