Add files using upload-large-folder tool

wandb/run-20240803_191521-sg37tylz/files/config.yaml

@@ -0,0 +1,335 @@
+wandb_version: 1
+
+sharding_strategy:
+  desc: null
+  value: FULL_SHARD
+checkpoint_type:
+  desc: null
+  value: LOCAL_STATE_DICT
+fsdp_activation_checkpointing:
+  desc: null
+  value: true
+fsdp_cpu_offload:
+  desc: null
+  value: false
+low_cpu_fsdp:
+  desc: null
+  value: false
+no_meta_device:
+  desc: null
+  value: false
+data_path:
+  desc: null
+  value: null
+split:
+  desc: null
+  value: 969, 30, 1
+train_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+valid_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+test_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+data_cache_path:
+  desc: null
+  value: null
+vocab_size:
+  desc: null
+  value: null
+vocab_file:
+  desc: null
+  value: null
+merge_file:
+  desc: null
+  value: null
+seq_length:
+  desc: null
+  value: 512
+num_workers:
+  desc: null
+  value: 2
+tokenizer_type:
+  desc: null
+  value: Llama2Tokenizer
+tokenizer_model:
+  desc: null
+  value: /share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3
+reset_position_ids:
+  desc: null
+  value: false
+reset_attention_mask:
+  desc: null
+  value: false
+eod_mask_loss:
+  desc: null
+  value: false
+retro_return_doc_ids:
+  desc: null
+  value: false
+short_seq_prob:
+  desc: null
+  value: 0.1
+vocab_extra_ids:
+  desc: null
+  value: 0
+seed:
+  desc: null
+  value: 1234
+use_mpi:
+  desc: null
+  value: false
+wandb_entity:
+  desc: null
+  value: iwakawa-koichi-q5-tohoku-nlp6723
+wandb_name:
+  desc: null
+  value: tiny-mistral-sample_train_2024-08-03-19:14:48
+wandb_project:
+  desc: null
+  value: llm_tutorial
+quantization:
+  desc: null
+  value: false
+use_freeze_layers:
+  desc: null
+  value: false
+freeze_layers:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+mixed_precision:
+  desc: null
+  value: true
+param_dtype:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: /work/llm_recipes/models/tiny-mistral-sample
+save:
+  desc: null
+  value: /work/llm_recipes/models/tiny-mistral-sample
+base_model:
+  desc: null
+  value: /share/pretrained_lm/custom/tiny-mistral
+use_better_transformer:
+  desc: null
+  value: false
+grad_clip_norm:
+  desc: null
+  value: 1.0
+eval_interval:
+  desc: null
+  value: 200
+save_interval:
+  desc: null
+  value: 200
+eval_iters:
+  desc: null
+  value: 10
+optimizer:
+  desc: null
+  value: adam
+lr:
+  desc: null
+  value: 2.0e-05
+lr_decay_style:
+  desc: null
+  value: cosine
+lr_decay_iters:
+  desc: null
+  value: 20000
+lr_warmup_iters:
+  desc: null
+  value: 500
+min_lr:
+  desc: null
+  value: 1.0e-06
+train_iters:
+  desc: null
+  value: 20000
+train_samples:
+  desc: null
+  value: null
+global_batch_size:
+  desc: null
+  value: 1600
+micro_batch_size:
+  desc: null
+  value: 40
+make_vocab_size_divisible_by:
+  desc: null
+  value: 128
+sliding_window_size:
+  desc: null
+  value: 4096
+skip_batch:
+  desc: null
+  value: null
+no_save_optimizer_state:
+  desc: null
+  value: false
+continual_pretraining:
+  desc: null
+  value: false
+instruction_tuning:
+  desc: null
+  value: false
+direct_preference_optimization:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.1
+hidden_dropout:
+  desc: null
+  value: 0.1
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_eps:
+  desc: null
+  value: 1.0e-06
+hf_transformer_model_dir:
+  desc: null
+  value: null
+instruction_train_data_path:
+  desc: null
+  value: null
+instruction_valid_data_path:
+  desc: null
+  value: null
+epoch:
+  desc: null
+  value: null
+instruction_dataset_size:
+  desc: null
+  value: null
+save_sampler_state:
+  desc: null
+  value: false
+label_smoothing:
+  desc: null
+  value: 0.0
+save_n_checkpoints:
+  desc: null
+  value: 10
+hf_repo_id:
+  desc: null
+  value: koichi12/tiny-mistral-sample
+create_public_hf_repo:
+  desc: null
+  value: false
+upload_all_checkpoints_to_hf:
+  desc: null
+  value: false
+hf_upload_retry_limit:
+  desc: null
+  value: 2
+exit_duration_in_mins:
+  desc: null
+  value: null
+source_key:
+  desc: null
+  value: null
+target_key:
+  desc: null
+  value: null
+attn_implementation:
+  desc: null
+  value: flash_attention_2
+efficient_instruction_tuning:
+  desc: null
+  value: false
+remove_padding_masking:
+  desc: null
+  value: false
+save_start_iter:
+  desc: null
+  value: null
+rank:
+  desc: null
+  value: 0
+world_size:
+  desc: null
+  value: 1
+padded_vocab_size:
+  desc: null
+  value: 32768
+gradient_accumulation_steps:
+  desc: null
+  value: 40
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.43.3
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1722680121.573481
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      2:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.43.3
+      8:
+      - 5
+      13: linux-x86_64
+activation_function:
+  desc: null
+  value: silu
+hidden_size:
+  desc: null
+  value: 256
+model_type:
+  desc: null
+  value: mistral
+max_position_embeddings:
+  desc: null
+  value: 512
+num_attention_heads:
+  desc: null
+  value: 4
+num_hidden_layers:
+  desc: null
+  value: 4
+model_architecture:
+  desc: null
+  value: MistralForCausalLM

wandb/run-20240803_191521-sg37tylz/files/output.log

@@ -0,0 +1,103 @@
+Created Hugging Face repository with ID koichi12/tiny-mistral-sample.
+Clearing GPU cache for all ranks
+--> Running with torch torch_distributed debug set to detail
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1.
+  warnings.warn(
+Let split = None
+Loading model state dict from /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/model.pt
+Loaded model state dict from /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/model.pt
+--> Model /share/pretrained_lm/custom/tiny-mistral
+--> /share/pretrained_lm/custom/tiny-mistral has 19.925248 Million params
+BFloat16 enabled for mixed precision - using bfSixteen policy
+--> applying fsdp activation checkpointing...
+ > datasets target sizes (minimum size):
+    train:      32000000
+    validation: 1616000
+    test:       16000
+> building train, validation, and test datasets for GPT ...
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+> finished creating GPT datasets ...
+Loading optimizer state dict from /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/optimizer.pt
+Loaded optimizer state dict from /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/optimizer.pt
+model info: FullyShardedDataParallel(
+  (_fsdp_wrapped_module): MistralForCausalLM(
+    (model): MistralModel(
+      (embed_tokens): Embedding(32768, 256)
+      (layers): ModuleList(
+        (0-3): 4 x FullyShardedDataParallel(
+          (_fsdp_wrapped_module): CheckpointWrapper(
+            (_checkpoint_wrapped_module): MistralDecoderLayer(
+              (self_attn): MistralFlashAttention2(
+                (q_proj): Linear(in_features=256, out_features=512, bias=False)
+                (k_proj): Linear(in_features=256, out_features=256, bias=False)
+                (v_proj): Linear(in_features=256, out_features=256, bias=False)
+                (o_proj): Linear(in_features=512, out_features=256, bias=False)
+                (rotary_emb): MistralRotaryEmbedding()
+              )
+              (mlp): MistralMLP(
+                (gate_proj): Linear(in_features=256, out_features=512, bias=False)
+                (up_proj): Linear(in_features=256, out_features=512, bias=False)
+                (down_proj): Linear(in_features=512, out_features=256, bias=False)
+                (act_fn): SiLU()
+              )
+              (input_layernorm): MistralRMSNorm()
+              (post_attention_layernorm): MistralRMSNorm()
+            )
+          )
+        )
+      )
+      (norm): MistralRMSNorm()
+    )
+    (lm_head): Linear(in_features=256, out_features=32768, bias=False)
+  )
+)
+model config: MistralConfig {
+  "_name_or_path": "/share/pretrained_lm/custom/tiny-mistral",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "label_smoothing": 0.0,
+  "max_position_embeddings": 512,
+  "model_type": "mistral",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 4,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.43.3",
+  "use_cache": false,
+  "vocab_size": 32768
+}
+Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000
+Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/model.pt
+Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/model.pt
+Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/optimizer.pt
+Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/optimizer.pt
+Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/scheduler.pt
+Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/scheduler.pt
+Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/rng.pt
+Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/rng.pt
+Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000, took 0.18s
+[rank0]:[2024-08-03 19:15:37,067] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling:  defaultdict(<class 'float'>, {})
+/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.
+  warnings.warn(
+/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.
+  warnings.warn(
+[rank0]:[2024-08-03 19:15:37,199] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling:  defaultdict(<class 'float'>, {'preprocessing': 0.0010301990000698424, 'preprocessing_with_comm': 0.009796595000352681, 'state_converting': 0.007119276000139507, <Type.ALL: 'all'>: 0.018263464000028762})

wandb/run-20240803_191521-sg37tylz/files/requirements.txt

@@ -0,0 +1,271 @@
+absl-py==2.1.0
+accelerate==0.33.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+apex==0.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+beautifulsoup4==4.12.3
+bleach==6.1.0
+blis==0.7.11
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+cmake==3.28.1
+colorama==0.4.6
+comm==0.2.1
+confection==0.1.4
+contourpy==1.2.0
+cubinlinker==0.3.0+2.g405ac64
+cuda-python==12.3.0rc4+9.gdb8c48a.dirty
+cudf==23.12.0
+cugraph-dgl==23.12.0
+cugraph-service-client==23.12.0
+cugraph-service-server==23.12.0
+cugraph==23.12.0
+cuml==23.12.0
+cupy-cuda12x==12.3.0
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.8
+dask-cuda==23.12.0
+dask-cudf==23.12.0
+dask==2023.11.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+distributed==2023.11.0
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+einops==0.7.0
+exceptiongroup==1.2.0
+execnet==2.0.2
+executing==2.0.1
+expecttest==0.1.3
+fastjsonschema==2.19.1
+fastrlock==0.8.2
+filelock==3.13.1
+flash-attn==2.4.2
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2023.12.2
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.27.0
+graphsurgeon==0.4.6
+grpcio==1.60.1
+huggingface-hub==0.24.5
+hypothesis==5.35.1
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+intel-openmp==2021.4.0
+ipadic==1.0.0
+ipykernel==6.29.2
+ipython-genutils==0.2.0
+ipython==8.21.0
+jedi==0.19.1
+jinja2==3.1.3
+joblib==1.3.2
+json5==0.9.14
+jsonnet==0.19.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.21.1
+jupyter-client==8.6.0
+jupyter-core==5.7.1
+jupyter-tensorboard==0.2.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==1.2.0
+jupyterlab==2.3.2
+jupytext==1.16.1
+kiwisolver==1.4.5
+langcodes==3.3.0
+lazy-loader==0.3
+librosa==0.10.1
+llvmlite==0.40.1
+locket==1.0.0
+logzero==1.7.0
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown==3.5.2
+markupsafe==2.1.4
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mecab-python3==1.0.6
+mistune==3.0.2
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mkl==2021.1.1
+mock==5.1.0
+more-itertools==9.1.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+murmurhash==1.0.10
+nbclient==0.9.0
+nbconvert==7.16.0
+nbformat==5.9.2
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.4.10
+numba==0.57.1+1.g1ff679645
+numpy==1.24.4
+nvfuser==0.1.4a0+d0bb811
+nvidia-dali-cuda120==1.34.0
+nvidia-pyindex==1.0.9
+nvtx==0.2.5
+oauthlib==3.2.2
+onnx==1.15.0rc2
+opencv==4.7.0
+optree==0.10.0
+packaging==23.2
+pandas==1.5.3
+pandocfilters==1.5.1
+parso==0.8.3
+partd==1.4.1
+peft==0.11.1
+pexpect==4.9.0
+pillow==10.2.0
+pip==24.0
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+polygraphy==0.49.4
+pooch==1.8.0
+portalocker==2.10.1
+preshed==3.0.9
+prettytable==3.9.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.24.4
+psutil==5.9.4
+ptxcompiler==0.8.1+2.g0d406d6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==14.0.1.dev0+gba5374836.d20240125
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pybind11-global==2.11.1
+pybind11==2.11.1
+pycocotools==2.0+nv0.8.0
+pycparser==2.21
+pydantic-core==2.16.2
+pydantic==2.6.1
+pygments==2.17.2
+pylibcugraph==23.12.0
+pylibcugraphops==23.12.0
+pylibraft==23.12.0
+pynvml==11.4.1
+pyparsing==3.1.1
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==13.0
+pytest-shard==0.1.2
+pytest-xdist==3.5.0
+pytest==8.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-hostlist==1.23.0
+pytorch-quantization==2.1.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.2
+raft-dask==23.12.0
+rapids-dask-dependency==23.12.1
+referencing==0.33.0
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests==2.31.0
+rich==13.7.0
+rmm==23.12.0
+rpds-py==0.17.1
+rsa==4.9
+sacrebleu==2.4.0
+safetensors==0.4.3
+scikit-learn==1.2.0
+scipy==1.12.0
+send2trash==1.8.2
+sentencepiece==0.1.99
+sentry-sdk==2.12.0
+setproctitle==1.3.3
+setuptools==68.2.2
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sphinx-glpi-theme==0.6
+srsly==2.4.8
+stack-data==0.6.3
+sympy==1.12
+tabulate==0.9.0
+tbb==2021.11.0
+tblib==3.0.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.9.0
+tensorrt==8.6.3
+terminado==0.18.0
+termplotlib==0.3.9
+thinc==8.2.3
+threadpoolctl==3.2.0
+thriftpy2==0.4.17
+tinycss2==1.2.1
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.1
+torch-tensorrt==2.3.0a0
+torch==2.3.0a0+ebedce2
+torchdata==0.7.1a0
+torchtext==0.17.0a0
+torchvision==0.18.0a0
+tornado==6.4
+tqdm==4.66.1
+traitlets==5.9.0
+transformer-engine==1.3.0+5b90b7f
+transformers==4.43.3
+treelite-runtime==3.9.1
+treelite==3.9.1
+triton==2.2.0+e28a256
+typer==0.9.0
+types-dataclasses==0.6.6
+typing-extensions==4.9.0
+ucx-py==0.35.0
+uff==0.6.9
+ujson==5.8.0
+urllib3==1.26.18
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.3.4
+webencodings==0.5.1
+werkzeug==3.0.1
+wheel==0.42.0
+xdoctest==1.0.2
+xgboost==1.7.6
+yarl==1.9.4
+zict==3.0.0
+zipp==3.17.0

wandb/run-20240803_191521-sg37tylz/files/wandb-metadata.json

@@ -0,0 +1,215 @@
+{
+    "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-08-03T10:15:22.462858",
+    "startedAt": "2024-08-03T10:15:21.560082",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--seq-length",
+        "512",
+        "--sliding-window-size",
+        "4096",
+        "--micro-batch-size",
+        "40",
+        "--global-batch-size",
+        "1600",
+        "--train-iters",
+        "20000",
+        "--tokenizer-type",
+        "Llama2Tokenizer",
+        "--tokenizer-model",
+        "/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3",
+        "--train-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--valid-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--test-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--lr",
+        "2e-5",
+        "--min-lr",
+        "1e-6",
+        "--lr-decay-style",
+        "cosine",
+        "--lr-warmup-iters",
+        "500",
+        "--lr-decay-iters",
+        "20000",
+        "--weight-decay",
+        "0.1",
+        "--grad-clip-norm",
+        "1.0",
+        "--optimizer",
+        "adam",
+        "--adam-beta1",
+        "0.9",
+        "--adam-beta2",
+        "0.95",
+        "--adam-eps",
+        "1e-6",
+        "--save-interval",
+        "200",
+        "--eval-interval",
+        "200",
+        "--eval-iters",
+        "10",
+        "--bf16",
+        "--mixed-precision",
+        "--base-model",
+        "/share/pretrained_lm/custom/tiny-mistral",
+        "--save",
+        "/work/llm_recipes/models/tiny-mistral-sample",
+        "--load",
+        "/work/llm_recipes/models/tiny-mistral-sample",
+        "--fsdp-activation-checkpointing",
+        "--sharding-strategy",
+        "FULL_SHARD",
+        "--checkpoint-type",
+        "LOCAL_STATE_DICT",
+        "--save-n-checkpoints",
+        "10",
+        "--hf-upload-retry-limit",
+        "2",
+        "--hf-repo-id",
+        "koichi12/tiny-mistral-sample",
+        "--wandb-entity",
+        "iwakawa-koichi-q5-tohoku-nlp6723",
+        "--wandb-project",
+        "llm_tutorial",
+        "--wandb-name",
+        "tiny-mistral-sample_train_2024-08-03-19:14:48"
+    ],
+    "state": "running",
+    "program": "/project/examples/finetuning.py",
+    "codePathLocal": "examples/finetuning.py",
+    "codePath": "examples/finetuning.py",
+    "git": {
+        "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git",
+        "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea"
+    },
+    "email": null,
+    "root": "/project",
+    "host": "gpu-koiwa-00",
+    "username": "koiwa",
+    "executable": "/usr/bin/python",
+    "cpu_count": 18,
+    "cpu_count_logical": 18,
+    "cpu_freq": {
+        "current": 2400.034,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 0.0625,
+            "used": 1.1444091796875e-05
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 56.48782730102539
+    }
+}

wandb/run-20240803_191521-sg37tylz/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 14}}

wandb/run-20240803_191521-sg37tylz/logs/debug-internal.log

@@ -0,0 +1,194 @@
+2024-08-03 19:15:21,574 INFO    StreamThr :9246 [internal.py:wandb_internal():86] W&B internal server running at pid: 9246, started at: 2024-08-03 19:15:21.573011
+2024-08-03 19:15:21,575 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: status
+2024-08-03 19:15:21,578 INFO    WriterThread:9246 [datastore.py:open_for_write():87] open: /project/wandb/run-20240803_191521-sg37tylz/run-sg37tylz.wandb
+2024-08-03 19:15:21,579 DEBUG   SenderThread:9246 [sender.py:send():382] send: header
+2024-08-03 19:15:21,755 DEBUG   SenderThread:9246 [sender.py:send():382] send: run
+2024-08-03 19:15:22,256 INFO    SenderThread:9246 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240803_191521-sg37tylz/files
+2024-08-03 19:15:22,257 INFO    SenderThread:9246 [sender.py:_start_run_threads():1136] run started: sg37tylz with start time 1722680121.573481
+2024-08-03 19:15:22,262 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: check_version
+2024-08-03 19:15:22,262 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: check_version
+2024-08-03 19:15:22,442 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: run_start
+2024-08-03 19:15:22,448 DEBUG   HandlerThread:9246 [system_info.py:__init__():27] System info init
+2024-08-03 19:15:22,448 DEBUG   HandlerThread:9246 [system_info.py:__init__():42] System info init done
+2024-08-03 19:15:22,448 INFO    HandlerThread:9246 [system_monitor.py:start():194] Starting system monitor
+2024-08-03 19:15:22,448 INFO    SystemMonitor:9246 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-08-03 19:15:22,449 INFO    HandlerThread:9246 [system_monitor.py:probe():214] Collecting system info
+2024-08-03 19:15:22,449 INFO    SystemMonitor:9246 [interfaces.py:start():190] Started cpu monitoring
+2024-08-03 19:15:22,450 INFO    SystemMonitor:9246 [interfaces.py:start():190] Started disk monitoring
+2024-08-03 19:15:22,450 INFO    SystemMonitor:9246 [interfaces.py:start():190] Started gpu monitoring
+2024-08-03 19:15:22,451 INFO    SystemMonitor:9246 [interfaces.py:start():190] Started memory monitoring
+2024-08-03 19:15:22,452 INFO    SystemMonitor:9246 [interfaces.py:start():190] Started network monitoring
+2024-08-03 19:15:22,462 DEBUG   HandlerThread:9246 [system_info.py:probe():151] Probing system
+2024-08-03 19:15:22,464 DEBUG   HandlerThread:9246 [system_info.py:_probe_git():136] Probing git
+2024-08-03 19:15:22,476 DEBUG   HandlerThread:9246 [system_info.py:_probe_git():144] Probing git done
+2024-08-03 19:15:22,476 DEBUG   HandlerThread:9246 [system_info.py:probe():199] Probing system done
+2024-08-03 19:15:22,476 DEBUG   HandlerThread:9246 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-03T10:15:22.462858', 'startedAt': '2024-08-03T10:15:21.560082', 'docker': None, 'cuda': None, 'args': ('--seq-length', '512', '--sliding-window-size', '4096', '--micro-batch-size', '40', '--global-batch-size', '1600', '--train-iters', '20000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/custom/tiny-mistral', '--save', '/work/llm_recipes/models/tiny-mistral-sample', '--load', '/work/llm_recipes/models/tiny-mistral-sample', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-mistral-sample', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-mistral-sample_train_2024-08-03-19:14:48'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.034, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}}
+2024-08-03 19:15:22,476 INFO    HandlerThread:9246 [system_monitor.py:probe():224] Finished collecting system info
+2024-08-03 19:15:22,476 INFO    HandlerThread:9246 [system_monitor.py:probe():227] Publishing system info
+2024-08-03 19:15:22,478 INFO    HandlerThread:9246 [system_monitor.py:probe():229] Finished publishing system info
+2024-08-03 19:15:22,505 DEBUG   SenderThread:9246 [sender.py:send():382] send: files
+2024-08-03 19:15:22,505 INFO    SenderThread:9246 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now
+2024-08-03 19:15:22,514 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: python_packages
+2024-08-03 19:15:22,514 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-03 19:15:22,514 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-03 19:15:22,514 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: python_packages
+2024-08-03 19:15:22,527 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: stop_status
+2024-08-03 19:15:22,755 DEBUG   SenderThread:9246 [sender.py:send():382] send: telemetry
+2024-08-03 19:15:23,258 INFO    Thread-12 :9246 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191521-sg37tylz/files/requirements.txt
+2024-08-03 19:15:23,258 INFO    Thread-12 :9246 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191521-sg37tylz/files/wandb-metadata.json
+2024-08-03 19:15:25,596 INFO    wandb-upload_0:9246 [upload_job.py:push():131] Uploaded file /tmp/tmp1lfwq_epwandb/7v7ji8nj-wandb-metadata.json
+2024-08-03 19:15:26,756 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:15:28,261 INFO    Thread-12 :9246 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191521-sg37tylz/files/output.log
+2024-08-03 19:15:30,262 INFO    Thread-12 :9246 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191521-sg37tylz/files/output.log
+2024-08-03 19:15:31,262 INFO    Thread-12 :9246 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191521-sg37tylz/files/output.log
+2024-08-03 19:15:31,904 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:15:33,263 INFO    Thread-12 :9246 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191521-sg37tylz/files/output.log
+2024-08-03 19:15:37,068 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:15:37,110 DEBUG   SenderThread:9246 [sender.py:send():382] send: config
+2024-08-03 19:15:37,111 DEBUG   SenderThread:9246 [sender.py:send():382] send: config
+2024-08-03 19:15:37,265 INFO    Thread-12 :9246 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191521-sg37tylz/files/output.log
+2024-08-03 19:15:37,316 DEBUG   SenderThread:9246 [sender.py:send():382] send: exit
+2024-08-03 19:15:37,316 INFO    SenderThread:9246 [sender.py:send_exit():589] handling exit code: 0
+2024-08-03 19:15:37,317 INFO    SenderThread:9246 [sender.py:send_exit():591] handling runtime: 14
+2024-08-03 19:15:37,318 INFO    SenderThread:9246 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:15:37,318 INFO    SenderThread:9246 [sender.py:send_exit():597] send defer
+2024-08-03 19:15:37,318 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:37,318 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 0
+2024-08-03 19:15:37,318 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:37,318 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 0
+2024-08-03 19:15:37,318 INFO    SenderThread:9246 [sender.py:transition_state():617] send defer: 1
+2024-08-03 19:15:37,319 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:37,319 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 1
+2024-08-03 19:15:37,319 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:37,319 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 1
+2024-08-03 19:15:37,319 INFO    SenderThread:9246 [sender.py:transition_state():617] send defer: 2
+2024-08-03 19:15:37,319 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:37,319 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 2
+2024-08-03 19:15:37,319 INFO    HandlerThread:9246 [system_monitor.py:finish():203] Stopping system monitor
+2024-08-03 19:15:37,319 DEBUG   SystemMonitor:9246 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-08-03 19:15:37,319 INFO    HandlerThread:9246 [interfaces.py:finish():202] Joined cpu monitor
+2024-08-03 19:15:37,320 DEBUG   SystemMonitor:9246 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-08-03 19:15:37,320 INFO    HandlerThread:9246 [interfaces.py:finish():202] Joined disk monitor
+2024-08-03 19:15:37,320 DEBUG   SystemMonitor:9246 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-08-03 19:15:37,351 INFO    HandlerThread:9246 [interfaces.py:finish():202] Joined gpu monitor
+2024-08-03 19:15:37,351 INFO    HandlerThread:9246 [interfaces.py:finish():202] Joined memory monitor
+2024-08-03 19:15:37,351 INFO    HandlerThread:9246 [interfaces.py:finish():202] Joined network monitor
+2024-08-03 19:15:37,352 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:37,352 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 2
+2024-08-03 19:15:37,352 INFO    SenderThread:9246 [sender.py:transition_state():617] send defer: 3
+2024-08-03 19:15:37,352 DEBUG   SenderThread:9246 [sender.py:send():382] send: stats
+2024-08-03 19:15:37,352 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:37,352 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 3
+2024-08-03 19:15:37,353 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:37,353 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 3
+2024-08-03 19:15:37,353 INFO    SenderThread:9246 [sender.py:transition_state():617] send defer: 4
+2024-08-03 19:15:37,353 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:37,353 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 4
+2024-08-03 19:15:37,353 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:37,353 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 4
+2024-08-03 19:15:37,353 INFO    SenderThread:9246 [sender.py:transition_state():617] send defer: 5
+2024-08-03 19:15:37,353 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:37,353 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 5
+2024-08-03 19:15:37,353 DEBUG   SenderThread:9246 [sender.py:send():382] send: summary
+2024-08-03 19:15:37,354 INFO    SenderThread:9246 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:15:37,354 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:37,354 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 5
+2024-08-03 19:15:37,354 INFO    SenderThread:9246 [sender.py:transition_state():617] send defer: 6
+2024-08-03 19:15:37,355 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:37,355 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 6
+2024-08-03 19:15:37,355 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:37,355 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 6
+2024-08-03 19:15:37,357 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:15:37,549 INFO    SenderThread:9246 [sender.py:transition_state():617] send defer: 7
+2024-08-03 19:15:37,550 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:37,550 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 7
+2024-08-03 19:15:37,550 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:37,550 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 7
+2024-08-03 19:15:38,267 INFO    Thread-12 :9246 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191521-sg37tylz/files/config.yaml
+2024-08-03 19:15:38,267 INFO    Thread-12 :9246 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191521-sg37tylz/files/wandb-summary.json
+2024-08-03 19:15:38,316 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-03 19:15:39,265 INFO    SenderThread:9246 [sender.py:transition_state():617] send defer: 8
+2024-08-03 19:15:39,265 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: poll_exit
+2024-08-03 19:15:39,265 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:39,265 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 8
+2024-08-03 19:15:39,265 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:39,265 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 8
+2024-08-03 19:15:39,266 INFO    SenderThread:9246 [job_builder.py:build():296] Attempting to build job artifact
+2024-08-03 19:15:39,266 INFO    SenderThread:9246 [job_builder.py:_get_source_type():426] is repo sourced job
+2024-08-03 19:15:39,267 INFO    Thread-12 :9246 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191521-sg37tylz/files/output.log
+2024-08-03 19:15:39,281 INFO    SenderThread:9246 [job_builder.py:build():402] adding wandb-job metadata file
+2024-08-03 19:15:39,289 INFO    SenderThread:9246 [sender.py:transition_state():617] send defer: 9
+2024-08-03 19:15:39,290 DEBUG   SenderThread:9246 [sender.py:send():382] send: artifact
+2024-08-03 19:15:39,290 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:39,291 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 9
+2024-08-03 19:15:39,317 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-03 19:15:40,500 INFO    wandb-upload_1:9246 [upload_job.py:push():86] Skipped uploading /singularity_home/.local/share/wandb/artifacts/staging/tmpftiijukc
+2024-08-03 19:15:40,873 INFO    wandb-upload_0:9246 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmp8adahfv9
+2024-08-03 19:15:42,354 INFO    SenderThread:9246 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA5MTk2NTkzOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA4NTUyMzkyNA==', 'versionIndex': 0}}}
+2024-08-03 19:15:42,354 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:42,354 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 9
+2024-08-03 19:15:42,354 INFO    SenderThread:9246 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-08-03 19:15:43,268 INFO    SenderThread:9246 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240803_191521-sg37tylz/files
+2024-08-03 19:15:43,268 INFO    SenderThread:9246 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191521-sg37tylz/files/requirements.txt requirements.txt
+2024-08-03 19:15:43,269 INFO    SenderThread:9246 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191521-sg37tylz/files/config.yaml config.yaml
+2024-08-03 19:15:43,270 INFO    SenderThread:9246 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191521-sg37tylz/files/wandb-metadata.json wandb-metadata.json
+2024-08-03 19:15:43,270 INFO    SenderThread:9246 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191521-sg37tylz/files/wandb-summary.json wandb-summary.json
+2024-08-03 19:15:43,271 INFO    SenderThread:9246 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191521-sg37tylz/files/output.log output.log
+2024-08-03 19:15:43,273 INFO    SenderThread:9246 [sender.py:transition_state():617] send defer: 10
+2024-08-03 19:15:43,273 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: poll_exit
+2024-08-03 19:15:43,273 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:43,275 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 10
+2024-08-03 19:15:43,275 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:43,275 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 10
+2024-08-03 19:15:43,275 INFO    SenderThread:9246 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-03 19:15:43,318 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-03 19:15:43,318 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: poll_exit
+2024-08-03 19:15:43,685 INFO    wandb-upload_1:9246 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191521-sg37tylz/files/requirements.txt
+2024-08-03 19:15:43,839 INFO    wandb-upload_0:9246 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191521-sg37tylz/files/config.yaml
+2024-08-03 19:15:43,870 INFO    wandb-upload_2:9246 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191521-sg37tylz/files/wandb-summary.json
+2024-08-03 19:15:43,873 INFO    wandb-upload_3:9246 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191521-sg37tylz/files/output.log
+2024-08-03 19:15:44,073 INFO    Thread-11 (_thread_body):9246 [sender.py:transition_state():617] send defer: 11
+2024-08-03 19:15:44,074 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:44,074 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 11
+2024-08-03 19:15:44,074 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:44,074 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 11
+2024-08-03 19:15:44,074 INFO    SenderThread:9246 [file_pusher.py:join():178] waiting for file pusher
+2024-08-03 19:15:44,074 INFO    SenderThread:9246 [sender.py:transition_state():617] send defer: 12
+2024-08-03 19:15:44,074 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:44,074 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 12
+2024-08-03 19:15:44,074 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:44,074 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 12
+2024-08-03 19:15:44,074 INFO    SenderThread:9246 [file_stream.py:finish():595] file stream finish called
+2024-08-03 19:15:44,248 INFO    SenderThread:9246 [file_stream.py:finish():599] file stream finish is done
+2024-08-03 19:15:44,248 INFO    SenderThread:9246 [sender.py:transition_state():617] send defer: 13
+2024-08-03 19:15:44,248 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:44,248 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 13
+2024-08-03 19:15:44,248 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:44,248 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 13
+2024-08-03 19:15:44,249 INFO    SenderThread:9246 [sender.py:transition_state():617] send defer: 14
+2024-08-03 19:15:44,249 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:15:44,249 DEBUG   SenderThread:9246 [sender.py:send():382] send: final
+2024-08-03 19:15:44,249 INFO    HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 14
+2024-08-03 19:15:44,249 DEBUG   SenderThread:9246 [sender.py:send():382] send: footer
+2024-08-03 19:15:44,249 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:15:44,249 INFO    SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 14
+2024-08-03 19:15:44,250 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-03 19:15:44,250 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: poll_exit
+2024-08-03 19:15:44,250 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-03 19:15:44,250 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: server_info
+2024-08-03 19:15:44,251 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: get_summary
+2024-08-03 19:15:44,251 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: poll_exit
+2024-08-03 19:15:44,251 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: server_info
+2024-08-03 19:15:44,252 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: sampled_history
+2024-08-03 19:15:44,252 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-03 19:15:44,253 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: job_info
+2024-08-03 19:15:44,413 DEBUG   SenderThread:9246 [sender.py:send_request():409] send_request: job_info
+2024-08-03 19:15:44,413 INFO    MainThread:9246 [wandb_run.py:_footer_history_summary_info():3866] rendering history
+2024-08-03 19:15:44,413 INFO    MainThread:9246 [wandb_run.py:_footer_history_summary_info():3898] rendering summary
+2024-08-03 19:15:44,414 INFO    MainThread:9246 [wandb_run.py:_footer_sync_info():3825] logging synced files
+2024-08-03 19:15:44,414 DEBUG   HandlerThread:9246 [handler.py:handle_request():146] handle_request: shutdown
+2024-08-03 19:15:44,414 INFO    HandlerThread:9246 [handler.py:finish():869] shutting down handler
+2024-08-03 19:15:45,253 INFO    WriterThread:9246 [datastore.py:close():296] close: /project/wandb/run-20240803_191521-sg37tylz/run-sg37tylz.wandb
+2024-08-03 19:15:45,413 INFO    SenderThread:9246 [sender.py:finish():1572] shutting down sender
+2024-08-03 19:15:45,413 INFO    SenderThread:9246 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-03 19:15:45,414 INFO    SenderThread:9246 [file_pusher.py:join():178] waiting for file pusher

wandb/run-20240803_191521-sg37tylz/logs/debug.log

@@ -0,0 +1,30 @@
+2024-08-03 19:15:21,565 INFO    MainThread:9173 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
+2024-08-03 19:15:21,566 INFO    MainThread:9173 [wandb_setup.py:_flush():76] Configure stats pid to 9173
+2024-08-03 19:15:21,566 INFO    MainThread:9173 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings
+2024-08-03 19:15:21,566 INFO    MainThread:9173 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings
+2024-08-03 19:15:21,566 INFO    MainThread:9173 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tuny llama sample'}
+2024-08-03 19:15:21,566 INFO    MainThread:9173 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-08-03 19:15:21,566 INFO    MainThread:9173 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'}
+2024-08-03 19:15:21,566 INFO    MainThread:9173 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240803_191521-sg37tylz/logs/debug.log
+2024-08-03 19:15:21,566 INFO    MainThread:9173 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240803_191521-sg37tylz/logs/debug-internal.log
+2024-08-03 19:15:21,566 INFO    MainThread:9173 [wandb_init.py:init():566] calling init triggers
+2024-08-03 19:15:21,566 INFO    MainThread:9173 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
+config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 512, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-mistral-sample_train_2024-08-03-19:14:48', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-mistral-sample', 'save': '/work/llm_recipes/models/tiny-mistral-sample', 'base_model': '/share/pretrained_lm/custom/tiny-mistral', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 1600, 'micro_batch_size': 40, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-mistral-sample', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32768, 'gradient_accumulation_steps': 40}
+2024-08-03 19:15:21,566 INFO    MainThread:9173 [wandb_init.py:init():616] starting backend
+2024-08-03 19:15:21,566 INFO    MainThread:9173 [wandb_init.py:init():620] setting up manager
+2024-08-03 19:15:21,571 INFO    MainThread:9173 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-08-03 19:15:21,573 INFO    MainThread:9173 [wandb_init.py:init():628] backend started and connected
+2024-08-03 19:15:21,578 INFO    MainThread:9173 [wandb_init.py:init():720] updated telemetry
+2024-08-03 19:15:21,751 INFO    MainThread:9173 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-08-03 19:15:22,261 INFO    MainThread:9173 [wandb_run.py:_on_init():2262] communicating current version
+2024-08-03 19:15:22,435 INFO    MainThread:9173 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-08-03 19:15:22,435 INFO    MainThread:9173 [wandb_init.py:init():804] starting run threads in backend
+2024-08-03 19:15:22,513 INFO    MainThread:9173 [wandb_run.py:_console_start():2241] atexit reg
+2024-08-03 19:15:22,514 INFO    MainThread:9173 [wandb_run.py:_redirect():2096] redirect: wrap_raw
+2024-08-03 19:15:22,514 INFO    MainThread:9173 [wandb_run.py:_redirect():2161] Wrapping output streams.
+2024-08-03 19:15:22,514 INFO    MainThread:9173 [wandb_run.py:_redirect():2186] Redirects installed.
+2024-08-03 19:15:22,514 INFO    MainThread:9173 [wandb_init.py:init():847] run started, returning control to user process
+2024-08-03 19:15:37,109 INFO    MainThread:9173 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'silu', 'hidden_size': 256, 'model_type': 'mistral', 'max_position_embeddings': 512, 'num_attention_heads': 4, 'num_hidden_layers': 4, 'model_architecture': 'MistralForCausalLM'}
+2024-08-03 19:15:37,110 INFO    MainThread:9173 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1}
+2024-08-03 19:15:45,414 WARNING MsgRouterThr:9173 [router.py:message_loop():77] message_loop has been closed

wandb/run-20240803_191815-jdwps0z3/files/config.yaml

@@ -0,0 +1,335 @@
+wandb_version: 1
+
+sharding_strategy:
+  desc: null
+  value: FULL_SHARD
+checkpoint_type:
+  desc: null
+  value: LOCAL_STATE_DICT
+fsdp_activation_checkpointing:
+  desc: null
+  value: true
+fsdp_cpu_offload:
+  desc: null
+  value: false
+low_cpu_fsdp:
+  desc: null
+  value: false
+no_meta_device:
+  desc: null
+  value: false
+data_path:
+  desc: null
+  value: null
+split:
+  desc: null
+  value: 969, 30, 1
+train_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+valid_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+test_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+data_cache_path:
+  desc: null
+  value: null
+vocab_size:
+  desc: null
+  value: null
+vocab_file:
+  desc: null
+  value: null
+merge_file:
+  desc: null
+  value: null
+seq_length:
+  desc: null
+  value: 512
+num_workers:
+  desc: null
+  value: 2
+tokenizer_type:
+  desc: null
+  value: Llama2Tokenizer
+tokenizer_model:
+  desc: null
+  value: /share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3
+reset_position_ids:
+  desc: null
+  value: false
+reset_attention_mask:
+  desc: null
+  value: false
+eod_mask_loss:
+  desc: null
+  value: false
+retro_return_doc_ids:
+  desc: null
+  value: false
+short_seq_prob:
+  desc: null
+  value: 0.1
+vocab_extra_ids:
+  desc: null
+  value: 0
+seed:
+  desc: null
+  value: 1234
+use_mpi:
+  desc: null
+  value: false
+wandb_entity:
+  desc: null
+  value: iwakawa-koichi-q5-tohoku-nlp6723
+wandb_name:
+  desc: null
+  value: tiny-mistral-sample2_train_2024-08-03-19:18:05
+wandb_project:
+  desc: null
+  value: llm_tutorial
+quantization:
+  desc: null
+  value: false
+use_freeze_layers:
+  desc: null
+  value: false
+freeze_layers:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+mixed_precision:
+  desc: null
+  value: true
+param_dtype:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: /work/llm_recipes/models/tiny-mistral-sample2
+save:
+  desc: null
+  value: /work/llm_recipes/models/tiny-mistral-sample2
+base_model:
+  desc: null
+  value: /share/pretrained_lm/custom/tiny-mistral
+use_better_transformer:
+  desc: null
+  value: false
+grad_clip_norm:
+  desc: null
+  value: 1.0
+eval_interval:
+  desc: null
+  value: 200
+save_interval:
+  desc: null
+  value: 200
+eval_iters:
+  desc: null
+  value: 10
+optimizer:
+  desc: null
+  value: adam
+lr:
+  desc: null
+  value: 2.0e-05
+lr_decay_style:
+  desc: null
+  value: cosine
+lr_decay_iters:
+  desc: null
+  value: 20000
+lr_warmup_iters:
+  desc: null
+  value: 500
+min_lr:
+  desc: null
+  value: 1.0e-06
+train_iters:
+  desc: null
+  value: 20000
+train_samples:
+  desc: null
+  value: null
+global_batch_size:
+  desc: null
+  value: 1600
+micro_batch_size:
+  desc: null
+  value: 40
+make_vocab_size_divisible_by:
+  desc: null
+  value: 128
+sliding_window_size:
+  desc: null
+  value: 4096
+skip_batch:
+  desc: null
+  value: null
+no_save_optimizer_state:
+  desc: null
+  value: false
+continual_pretraining:
+  desc: null
+  value: false
+instruction_tuning:
+  desc: null
+  value: false
+direct_preference_optimization:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.1
+hidden_dropout:
+  desc: null
+  value: 0.1
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_eps:
+  desc: null
+  value: 1.0e-06
+hf_transformer_model_dir:
+  desc: null
+  value: null
+instruction_train_data_path:
+  desc: null
+  value: null
+instruction_valid_data_path:
+  desc: null
+  value: null
+epoch:
+  desc: null
+  value: null
+instruction_dataset_size:
+  desc: null
+  value: null
+save_sampler_state:
+  desc: null
+  value: false
+label_smoothing:
+  desc: null
+  value: 0.0
+save_n_checkpoints:
+  desc: null
+  value: 10
+hf_repo_id:
+  desc: null
+  value: koichi12/tiny-mistral-sample2
+create_public_hf_repo:
+  desc: null
+  value: false
+upload_all_checkpoints_to_hf:
+  desc: null
+  value: false
+hf_upload_retry_limit:
+  desc: null
+  value: 2
+exit_duration_in_mins:
+  desc: null
+  value: null
+source_key:
+  desc: null
+  value: null
+target_key:
+  desc: null
+  value: null
+attn_implementation:
+  desc: null
+  value: flash_attention_2
+efficient_instruction_tuning:
+  desc: null
+  value: false
+remove_padding_masking:
+  desc: null
+  value: false
+save_start_iter:
+  desc: null
+  value: null
+rank:
+  desc: null
+  value: 0
+world_size:
+  desc: null
+  value: 1
+padded_vocab_size:
+  desc: null
+  value: 32768
+gradient_accumulation_steps:
+  desc: null
+  value: 40
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.43.3
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1722680295.872336
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      2:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.43.3
+      8:
+      - 5
+      13: linux-x86_64
+activation_function:
+  desc: null
+  value: silu
+hidden_size:
+  desc: null
+  value: 256
+model_type:
+  desc: null
+  value: mistral
+max_position_embeddings:
+  desc: null
+  value: 512
+num_attention_heads:
+  desc: null
+  value: 4
+num_hidden_layers:
+  desc: null
+  value: 4
+model_architecture:
+  desc: null
+  value: MistralForCausalLM

wandb/run-20240803_191815-jdwps0z3/files/output.log

@@ -0,0 +1,239 @@
+Created Hugging Face repository with ID koichi12/tiny-mistral-sample2.
+Clearing GPU cache for all ranks
+--> Running with torch torch_distributed debug set to detail
+File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt
+File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt
+File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt
+No checkpoint found in /work/llm_recipes/models/tiny-mistral-sample2, skipping model loading
+--> Model /share/pretrained_lm/custom/tiny-mistral
+--> /share/pretrained_lm/custom/tiny-mistral has 19.925248 Million params
+BFloat16 enabled for mixed precision - using bfSixteen policy
+--> applying fsdp activation checkpointing...
+ > datasets target sizes (minimum size):
+    train:      32000000
+    validation: 1616000
+    test:       16000
+> building train, validation, and test datasets for GPT ...
+> finished creating GPT datasets ...
+File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt
+No checkpoint found in /work/llm_recipes/models/tiny-mistral-sample2, skipping optimizer loading
+File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt
+model info: FullyShardedDataParallel(
+  (_fsdp_wrapped_module): MistralForCausalLM(
+    (model): MistralModel(
+      (embed_tokens): Embedding(32768, 256)
+      (layers): ModuleList(
+        (0-3): 4 x FullyShardedDataParallel(
+          (_fsdp_wrapped_module): CheckpointWrapper(
+            (_checkpoint_wrapped_module): MistralDecoderLayer(
+              (self_attn): MistralFlashAttention2(
+                (q_proj): Linear(in_features=256, out_features=512, bias=False)
+                (k_proj): Linear(in_features=256, out_features=256, bias=False)
+                (v_proj): Linear(in_features=256, out_features=256, bias=False)
+                (o_proj): Linear(in_features=512, out_features=256, bias=False)
+                (rotary_emb): MistralRotaryEmbedding()
+              )
+              (mlp): MistralMLP(
+                (gate_proj): Linear(in_features=256, out_features=512, bias=False)
+                (up_proj): Linear(in_features=256, out_features=512, bias=False)
+                (down_proj): Linear(in_features=512, out_features=256, bias=False)
+                (act_fn): SiLU()
+              )
+              (input_layernorm): MistralRMSNorm()
+              (post_attention_layernorm): MistralRMSNorm()
+            )
+          )
+        )
+      )
+      (norm): MistralRMSNorm()
+    )
+    (lm_head): Linear(in_features=256, out_features=32768, bias=False)
+  )
+)
+model config: MistralConfig {
+  "_name_or_path": "/share/pretrained_lm/custom/tiny-mistral",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "intermediate_size": 512,
+  "label_smoothing": 0.0,
+  "max_position_embeddings": 512,
+  "model_type": "mistral",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 4,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.43.3",
+  "use_cache": false,
+  "vocab_size": 32768
+}
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1.
+  warnings.warn(
+Let split = None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+------------------------------------------------------------------
+iteration: 1 , TFLOPS: 7.474408749953364, Tokens per sec: 105583.91620199519, Loss: 10.45809555053711
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 2 , TFLOPS: 19.626322889381036, Tokens per sec: 277242.53524650185, Loss: 10.456090927124023
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 3 , TFLOPS: 19.74558354343518, Tokens per sec: 278927.2179184158, Loss: 10.458115577697754
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 4 , TFLOPS: 19.588618586475842, Tokens per sec: 276709.92215407215, Loss: 10.456512451171875
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 5 , TFLOPS: 19.72619917981179, Tokens per sec: 278653.39331329847, Loss: 10.458467483520508
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 6 , TFLOPS: 19.627595003583973, Tokens per sec: 277260.5052029086, Loss: 10.457172393798828
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 7 , TFLOPS: 19.77370988376223, Tokens per sec: 279324.53217557067, Loss: 10.457324981689453
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 8 , TFLOPS: 19.77766998452317, Tokens per sec: 279380.47278049105, Loss: 10.457306861877441
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 9 , TFLOPS: 19.749958254856136, Tokens per sec: 278989.01533671736, Loss: 10.457944869995117
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 10 , TFLOPS: 19.752309517490676, Tokens per sec: 279022.22940424824, Loss: 10.45663833618164
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 11 , TFLOPS: 19.74392548352723, Tokens per sec: 278903.7960713861, Loss: 10.456352233886719
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 12 , TFLOPS: 19.758394357809003, Tokens per sec: 279108.18419903744, Loss: 10.455950736999512
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 13 , TFLOPS: 19.729624964309398, Tokens per sec: 278701.78613678756, Loss: 10.45804214477539
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 14 , TFLOPS: 19.626730348672773, Tokens per sec: 277248.29103925463, Loss: 10.457955360412598
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 15 , TFLOPS: 19.724026466287597, Tokens per sec: 278622.7014404904, Loss: 10.459123611450195
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 16 , TFLOPS: 19.108803574684035, Tokens per sec: 269932.0284514028, Loss: 10.45695686340332
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 17 , TFLOPS: 19.73958477407385, Tokens per sec: 278842.4789667809, Loss: 10.456724166870117
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 18 , TFLOPS: 19.47597897335848, Tokens per sec: 275118.76867688465, Loss: 10.456052780151367
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 19 , TFLOPS: 19.72614156699911, Tokens per sec: 278652.57947148307, Loss: 10.458043098449707
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 20 , TFLOPS: 19.701494428407866, Tokens per sec: 278304.4126127127, Loss: 10.455198287963867
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 21 , TFLOPS: 19.74596953791029, Tokens per sec: 278932.6704979679, Loss: 10.457011222839355
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 22 , TFLOPS: 19.615963035173014, Tokens per sec: 277096.1913663158, Loss: 10.457304954528809
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 23 , TFLOPS: 19.718852381531324, Tokens per sec: 278549.61202972836, Loss: 10.456293106079102
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 24 , TFLOPS: 19.67743842750437, Tokens per sec: 277964.5961979942, Loss: 10.456793785095215
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 25 , TFLOPS: 19.72792104268366, Tokens per sec: 278677.7164445664, Loss: 10.45568561553955
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 26 , TFLOPS: 19.766049190998213, Tokens per sec: 279216.31680096325, Loss: 10.455270767211914
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 27 , TFLOPS: 19.748005203155174, Tokens per sec: 278961.4264191145, Loss: 10.456525802612305
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 28 , TFLOPS: 19.788746629218007, Tokens per sec: 279536.9420831989, Loss: 10.458827018737793
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 29 , TFLOPS: 19.64595613343327, Tokens per sec: 277519.8755504821, Loss: 10.454755783081055
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 30 , TFLOPS: 19.791751212574006, Tokens per sec: 279579.38499579503, Loss: 10.455424308776855
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 31 , TFLOPS: 19.647830180890995, Tokens per sec: 277546.34844972467, Loss: 10.455726623535156
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 32 , TFLOPS: 19.108735127334096, Tokens per sec: 269931.06156030786, Loss: 10.456134796142578
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 33 , TFLOPS: 19.790956814139804, Tokens per sec: 279568.16327906615, Loss: 10.45483684539795
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 34 , TFLOPS: 19.809776481063633, Tokens per sec: 279834.0109470101, Loss: 10.455580711364746
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 35 , TFLOPS: 19.78921404344017, Tokens per sec: 279543.5448026535, Loss: 10.4553861618042
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 36 , TFLOPS: 19.760295167208643, Tokens per sec: 279135.03513896884, Loss: 10.455459594726562
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 37 , TFLOPS: 19.77547756611738, Tokens per sec: 279349.502555423, Loss: 10.45541000366211
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 38 , TFLOPS: 19.777886920468827, Tokens per sec: 279383.53722979716, Loss: 10.455677032470703
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 39 , TFLOPS: 19.686203609180314, Tokens per sec: 278088.4136447687, Loss: 10.454801559448242
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 40 , TFLOPS: 19.733730730492624, Tokens per sec: 278759.7844196133, Loss: 10.45442008972168
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 41 , TFLOPS: 19.781936717382273, Tokens per sec: 279440.74488758645, Loss: 10.453974723815918
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 42 , TFLOPS: 19.713193575703478, Tokens per sec: 278469.67542198877, Loss: 10.454327583312988
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 43 , TFLOPS: 19.765683899965286, Tokens per sec: 279211.1566793938, Loss: 10.453243255615234
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 44 , TFLOPS: 19.737930705062368, Tokens per sec: 278819.11349537794, Loss: 10.452882766723633
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 45 , TFLOPS: 19.768669599163157, Tokens per sec: 279253.3328333156, Loss: 10.452847480773926
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 46 , TFLOPS: 19.195477136883916, Tokens per sec: 271156.3840405051, Loss: 10.452914237976074
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 47 , TFLOPS: 19.78773782868223, Tokens per sec: 279522.69171038724, Loss: 10.452442169189453
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 48 , TFLOPS: 19.79785807451266, Tokens per sec: 279665.6508692251, Loss: 10.452777862548828
+------------------------------------------------------------------

wandb/run-20240803_191815-jdwps0z3/files/requirements.txt

@@ -0,0 +1,271 @@
+absl-py==2.1.0
+accelerate==0.33.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+apex==0.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+beautifulsoup4==4.12.3
+bleach==6.1.0
+blis==0.7.11
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+cmake==3.28.1
+colorama==0.4.6
+comm==0.2.1
+confection==0.1.4
+contourpy==1.2.0
+cubinlinker==0.3.0+2.g405ac64
+cuda-python==12.3.0rc4+9.gdb8c48a.dirty
+cudf==23.12.0
+cugraph-dgl==23.12.0
+cugraph-service-client==23.12.0
+cugraph-service-server==23.12.0
+cugraph==23.12.0
+cuml==23.12.0
+cupy-cuda12x==12.3.0
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.8
+dask-cuda==23.12.0
+dask-cudf==23.12.0
+dask==2023.11.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+distributed==2023.11.0
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+einops==0.7.0
+exceptiongroup==1.2.0
+execnet==2.0.2
+executing==2.0.1
+expecttest==0.1.3
+fastjsonschema==2.19.1
+fastrlock==0.8.2
+filelock==3.13.1
+flash-attn==2.4.2
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2023.12.2
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.27.0
+graphsurgeon==0.4.6
+grpcio==1.60.1
+huggingface-hub==0.24.5
+hypothesis==5.35.1
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+intel-openmp==2021.4.0
+ipadic==1.0.0
+ipykernel==6.29.2
+ipython-genutils==0.2.0
+ipython==8.21.0
+jedi==0.19.1
+jinja2==3.1.3
+joblib==1.3.2
+json5==0.9.14
+jsonnet==0.19.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.21.1
+jupyter-client==8.6.0
+jupyter-core==5.7.1
+jupyter-tensorboard==0.2.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==1.2.0
+jupyterlab==2.3.2
+jupytext==1.16.1
+kiwisolver==1.4.5
+langcodes==3.3.0
+lazy-loader==0.3
+librosa==0.10.1
+llvmlite==0.40.1
+locket==1.0.0
+logzero==1.7.0
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown==3.5.2
+markupsafe==2.1.4
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mecab-python3==1.0.6
+mistune==3.0.2
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mkl==2021.1.1
+mock==5.1.0
+more-itertools==9.1.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+murmurhash==1.0.10
+nbclient==0.9.0
+nbconvert==7.16.0
+nbformat==5.9.2
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.4.10
+numba==0.57.1+1.g1ff679645
+numpy==1.24.4
+nvfuser==0.1.4a0+d0bb811
+nvidia-dali-cuda120==1.34.0
+nvidia-pyindex==1.0.9
+nvtx==0.2.5
+oauthlib==3.2.2
+onnx==1.15.0rc2
+opencv==4.7.0
+optree==0.10.0
+packaging==23.2
+pandas==1.5.3
+pandocfilters==1.5.1
+parso==0.8.3
+partd==1.4.1
+peft==0.11.1
+pexpect==4.9.0
+pillow==10.2.0
+pip==24.0
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+polygraphy==0.49.4
+pooch==1.8.0
+portalocker==2.10.1
+preshed==3.0.9
+prettytable==3.9.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.24.4
+psutil==5.9.4
+ptxcompiler==0.8.1+2.g0d406d6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==14.0.1.dev0+gba5374836.d20240125
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pybind11-global==2.11.1
+pybind11==2.11.1
+pycocotools==2.0+nv0.8.0
+pycparser==2.21
+pydantic-core==2.16.2
+pydantic==2.6.1
+pygments==2.17.2
+pylibcugraph==23.12.0
+pylibcugraphops==23.12.0
+pylibraft==23.12.0
+pynvml==11.4.1
+pyparsing==3.1.1
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==13.0
+pytest-shard==0.1.2
+pytest-xdist==3.5.0
+pytest==8.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-hostlist==1.23.0
+pytorch-quantization==2.1.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.2
+raft-dask==23.12.0
+rapids-dask-dependency==23.12.1
+referencing==0.33.0
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests==2.31.0
+rich==13.7.0
+rmm==23.12.0
+rpds-py==0.17.1
+rsa==4.9
+sacrebleu==2.4.0
+safetensors==0.4.3
+scikit-learn==1.2.0
+scipy==1.12.0
+send2trash==1.8.2
+sentencepiece==0.1.99
+sentry-sdk==2.12.0
+setproctitle==1.3.3
+setuptools==68.2.2
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sphinx-glpi-theme==0.6
+srsly==2.4.8
+stack-data==0.6.3
+sympy==1.12
+tabulate==0.9.0
+tbb==2021.11.0
+tblib==3.0.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.9.0
+tensorrt==8.6.3
+terminado==0.18.0
+termplotlib==0.3.9
+thinc==8.2.3
+threadpoolctl==3.2.0
+thriftpy2==0.4.17
+tinycss2==1.2.1
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.1
+torch-tensorrt==2.3.0a0
+torch==2.3.0a0+ebedce2
+torchdata==0.7.1a0
+torchtext==0.17.0a0
+torchvision==0.18.0a0
+tornado==6.4
+tqdm==4.66.1
+traitlets==5.9.0
+transformer-engine==1.3.0+5b90b7f
+transformers==4.43.3
+treelite-runtime==3.9.1
+treelite==3.9.1
+triton==2.2.0+e28a256
+typer==0.9.0
+types-dataclasses==0.6.6
+typing-extensions==4.9.0
+ucx-py==0.35.0
+uff==0.6.9
+ujson==5.8.0
+urllib3==1.26.18
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.3.4
+webencodings==0.5.1
+werkzeug==3.0.1
+wheel==0.42.0
+xdoctest==1.0.2
+xgboost==1.7.6
+yarl==1.9.4
+zict==3.0.0
+zipp==3.17.0

wandb/run-20240803_191815-jdwps0z3/files/wandb-metadata.json

@@ -0,0 +1,215 @@
+{
+    "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-08-03T10:18:16.495932",
+    "startedAt": "2024-08-03T10:18:15.860061",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--seq-length",
+        "512",
+        "--sliding-window-size",
+        "4096",
+        "--micro-batch-size",
+        "40",
+        "--global-batch-size",
+        "1600",
+        "--train-iters",
+        "20000",
+        "--tokenizer-type",
+        "Llama2Tokenizer",
+        "--tokenizer-model",
+        "/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3",
+        "--train-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--valid-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--test-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--lr",
+        "2e-5",
+        "--min-lr",
+        "1e-6",
+        "--lr-decay-style",
+        "cosine",
+        "--lr-warmup-iters",
+        "500",
+        "--lr-decay-iters",
+        "20000",
+        "--weight-decay",
+        "0.1",
+        "--grad-clip-norm",
+        "1.0",
+        "--optimizer",
+        "adam",
+        "--adam-beta1",
+        "0.9",
+        "--adam-beta2",
+        "0.95",
+        "--adam-eps",
+        "1e-6",
+        "--save-interval",
+        "200",
+        "--eval-interval",
+        "200",
+        "--eval-iters",
+        "10",
+        "--bf16",
+        "--mixed-precision",
+        "--base-model",
+        "/share/pretrained_lm/custom/tiny-mistral",
+        "--save",
+        "/work/llm_recipes/models/tiny-mistral-sample2",
+        "--load",
+        "/work/llm_recipes/models/tiny-mistral-sample2",
+        "--fsdp-activation-checkpointing",
+        "--sharding-strategy",
+        "FULL_SHARD",
+        "--checkpoint-type",
+        "LOCAL_STATE_DICT",
+        "--save-n-checkpoints",
+        "10",
+        "--hf-upload-retry-limit",
+        "2",
+        "--hf-repo-id",
+        "koichi12/tiny-mistral-sample2",
+        "--wandb-entity",
+        "iwakawa-koichi-q5-tohoku-nlp6723",
+        "--wandb-project",
+        "llm_tutorial",
+        "--wandb-name",
+        "tiny-mistral-sample2_train_2024-08-03-19:18:05"
+    ],
+    "state": "running",
+    "program": "/project/examples/finetuning.py",
+    "codePathLocal": "examples/finetuning.py",
+    "codePath": "examples/finetuning.py",
+    "git": {
+        "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git",
+        "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea"
+    },
+    "email": null,
+    "root": "/project",
+    "host": "gpu-koiwa-00",
+    "username": "koiwa",
+    "executable": "/usr/bin/python",
+    "cpu_count": 18,
+    "cpu_count_logical": 18,
+    "cpu_freq": {
+        "current": 2400.034,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 0.0625,
+            "used": 1.1444091796875e-05
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 56.48782730102539
+    }
+}

wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json

@@ -0,0 +1 @@
+{"training/loss": 10.452777862548828, "training/perplexity": 34640.467638738424, "utils/batch_size": 40, "utils/global_batch_size": 1600, "utils/seq_len": 513, "utils/gradient_accumulation_steps": 40, "utils/iteration": 48, "optimizer/lr": 2.8240000000000004e-06, "optimizer/variance_l2": 0.0037958921404718368, "optimizer/variance_sqrt_l2": 0.9574357404136762, "optimizer/momentum_l2": 0.9945472829864366, "optimizer/weight_l2": 101.38293742045552, "optimizer/variance_l1": 0.91748046875, "optimizer/variance_sqrt_l1": 723.0, "optimizer/momentum_l1": 740.5, "optimizer/weight_l1": 320512.0, "optimizer/variance_abs_max": 0.0003185272216796875, "optimizer/variance_sqrt_abs_max": 0.017822265625, "optimizer/momentum_abs_max": 0.0184326171875, "optimizer/weight_abs_max": 1.0, "stats/1_iteration_time": 2.9349331869998423, "stats/tokens_per_sec": 279665.6508692251, "stats/tokens_per_sec_per_gpu": 279665.6508692251, "stats/tflops": 19.79785807451266, "_timestamp": 1722680446.903461, "_runtime": 151.03112506866455, "_step": 48, "_wandb": {"runtime": 151}}

wandb/run-20240803_191815-jdwps0z3/logs/debug-internal.log

@@ -0,0 +1,524 @@
+2024-08-03 19:18:15,873 INFO    StreamThr :9504 [internal.py:wandb_internal():86] W&B internal server running at pid: 9504, started at: 2024-08-03 19:18:15.872881
+2024-08-03 19:18:15,875 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status
+2024-08-03 19:18:15,877 INFO    WriterThread:9504 [datastore.py:open_for_write():87] open: /project/wandb/run-20240803_191815-jdwps0z3/run-jdwps0z3.wandb
+2024-08-03 19:18:15,878 DEBUG   SenderThread:9504 [sender.py:send():382] send: header
+2024-08-03 19:18:15,892 DEBUG   SenderThread:9504 [sender.py:send():382] send: run
+2024-08-03 19:18:16,382 INFO    SenderThread:9504 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240803_191815-jdwps0z3/files
+2024-08-03 19:18:16,382 INFO    SenderThread:9504 [sender.py:_start_run_threads():1136] run started: jdwps0z3 with start time 1722680295.872336
+2024-08-03 19:18:16,387 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: check_version
+2024-08-03 19:18:16,387 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: check_version
+2024-08-03 19:18:16,477 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: run_start
+2024-08-03 19:18:16,483 DEBUG   HandlerThread:9504 [system_info.py:__init__():27] System info init
+2024-08-03 19:18:16,483 DEBUG   HandlerThread:9504 [system_info.py:__init__():42] System info init done
+2024-08-03 19:18:16,483 INFO    HandlerThread:9504 [system_monitor.py:start():194] Starting system monitor
+2024-08-03 19:18:16,483 INFO    SystemMonitor:9504 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-08-03 19:18:16,483 INFO    HandlerThread:9504 [system_monitor.py:probe():214] Collecting system info
+2024-08-03 19:18:16,484 INFO    SystemMonitor:9504 [interfaces.py:start():190] Started cpu monitoring
+2024-08-03 19:18:16,485 INFO    SystemMonitor:9504 [interfaces.py:start():190] Started disk monitoring
+2024-08-03 19:18:16,486 INFO    SystemMonitor:9504 [interfaces.py:start():190] Started gpu monitoring
+2024-08-03 19:18:16,486 INFO    SystemMonitor:9504 [interfaces.py:start():190] Started memory monitoring
+2024-08-03 19:18:16,487 INFO    SystemMonitor:9504 [interfaces.py:start():190] Started network monitoring
+2024-08-03 19:18:16,495 DEBUG   HandlerThread:9504 [system_info.py:probe():151] Probing system
+2024-08-03 19:18:16,497 DEBUG   HandlerThread:9504 [system_info.py:_probe_git():136] Probing git
+2024-08-03 19:18:16,509 DEBUG   HandlerThread:9504 [system_info.py:_probe_git():144] Probing git done
+2024-08-03 19:18:16,509 DEBUG   HandlerThread:9504 [system_info.py:probe():199] Probing system done
+2024-08-03 19:18:16,509 DEBUG   HandlerThread:9504 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-03T10:18:16.495932', 'startedAt': '2024-08-03T10:18:15.860061', 'docker': None, 'cuda': None, 'args': ('--seq-length', '512', '--sliding-window-size', '4096', '--micro-batch-size', '40', '--global-batch-size', '1600', '--train-iters', '20000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/custom/tiny-mistral', '--save', '/work/llm_recipes/models/tiny-mistral-sample2', '--load', '/work/llm_recipes/models/tiny-mistral-sample2', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-mistral-sample2', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-mistral-sample2_train_2024-08-03-19:18:05'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.034, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}}
+2024-08-03 19:18:16,509 INFO    HandlerThread:9504 [system_monitor.py:probe():224] Finished collecting system info
+2024-08-03 19:18:16,509 INFO    HandlerThread:9504 [system_monitor.py:probe():227] Publishing system info
+2024-08-03 19:18:16,510 INFO    HandlerThread:9504 [system_monitor.py:probe():229] Finished publishing system info
+2024-08-03 19:18:16,516 DEBUG   SenderThread:9504 [sender.py:send():382] send: files
+2024-08-03 19:18:16,516 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now
+2024-08-03 19:18:16,525 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: python_packages
+2024-08-03 19:18:16,526 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-03 19:18:16,526 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-03 19:18:16,526 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: python_packages
+2024-08-03 19:18:16,528 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: stop_status
+2024-08-03 19:18:16,829 DEBUG   SenderThread:9504 [sender.py:send():382] send: telemetry
+2024-08-03 19:18:17,199 INFO    wandb-upload_0:9504 [upload_job.py:push():131] Uploaded file /tmp/tmp76a9qs5lwandb/q74vxtjq-wandb-metadata.json
+2024-08-03 19:18:17,385 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-metadata.json
+2024-08-03 19:18:17,386 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191815-jdwps0z3/files/requirements.txt
+2024-08-03 19:18:17,386 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:18:17,990 DEBUG   SenderThread:9504 [sender.py:send():382] send: config
+2024-08-03 19:18:17,991 DEBUG   SenderThread:9504 [sender.py:send():382] send: config
+2024-08-03 19:18:19,388 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:18:20,991 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:18:25,992 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:18:28,096 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:18:30,394 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:18:31,059 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:18:31,062 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:18:31,063 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:18:31,063 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:18:31,064 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:18:31,395 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:18:31,525 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-03 19:18:31,526 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: stop_status
+2024-08-03 19:18:31,568 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-03 19:18:34,003 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:18:34,005 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:18:34,005 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:18:34,006 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:18:34,397 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:18:34,397 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:18:36,398 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:18:36,971 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:18:36,974 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:18:36,975 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:18:36,976 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:18:36,977 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:18:37,399 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:18:39,919 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:18:39,921 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:18:39,921 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:18:39,922 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:18:40,401 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:18:40,401 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:18:42,402 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:18:42,881 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:18:42,883 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:18:42,883 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:18:42,884 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:18:42,885 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:18:43,403 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:18:44,403 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:18:45,821 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:18:45,823 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:18:45,824 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:18:45,825 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:18:46,405 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:18:46,525 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-03 19:18:46,525 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: stop_status
+2024-08-03 19:18:46,526 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-03 19:18:48,406 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:18:48,697 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:18:48,761 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:18:48,885 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:18:48,885 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:18:48,887 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:18:49,407 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:18:49,407 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/config.yaml
+2024-08-03 19:18:50,407 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:18:51,705 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:18:51,707 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:18:51,708 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:18:51,709 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:18:52,409 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:18:54,410 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:18:54,648 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:18:54,651 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:18:54,651 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:18:54,651 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:18:54,652 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:18:55,411 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:18:56,412 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:18:57,593 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:18:57,596 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:18:57,596 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:18:57,597 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:18:58,413 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:00,414 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:00,536 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:00,540 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:00,540 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:00,541 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:19:00,542 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:01,415 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:01,525 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-03 19:19:01,525 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: stop_status
+2024-08-03 19:19:01,526 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-03 19:19:02,416 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:03,483 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:03,484 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:03,485 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:03,486 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:04,417 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:06,418 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:06,446 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:06,448 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:06,448 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:06,448 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:19:06,449 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:07,420 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:08,420 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:09,393 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:09,396 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:09,396 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:09,397 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:09,421 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:10,422 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:12,398 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:19:12,436 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:12,438 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:12,438 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:12,440 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:13,424 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:14,424 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:15,381 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:15,383 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:15,384 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:15,385 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:15,425 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:16,426 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:16,487 DEBUG   SystemMonitor:9504 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-08-03 19:19:16,489 DEBUG   SenderThread:9504 [sender.py:send():382] send: stats
+2024-08-03 19:19:16,525 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-03 19:19:16,525 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: stop_status
+2024-08-03 19:19:16,526 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-03 19:19:17,757 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:19:18,366 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:18,368 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:18,368 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:18,369 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:18,427 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:20,428 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:21,314 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:21,316 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:21,317 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:21,318 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:21,429 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:22,430 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:23,318 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:19:24,265 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:24,267 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:24,268 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:24,269 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:24,431 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:26,432 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:27,210 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:27,212 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:27,213 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:27,214 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:27,433 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:28,434 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:29,214 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:19:30,174 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:30,175 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:30,176 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:30,177 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:30,435 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:31,525 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-03 19:19:31,525 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: stop_status
+2024-08-03 19:19:31,527 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-03 19:19:32,436 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:33,125 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:33,126 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:33,126 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:33,127 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:33,437 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:34,438 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:35,169 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:19:36,079 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:36,081 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:36,082 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:36,084 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:36,439 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:38,441 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:39,026 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:39,029 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:39,029 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:39,030 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:39,442 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:40,443 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:41,031 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:19:41,968 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:41,970 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:41,970 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:41,972 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:42,444 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:44,445 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:44,912 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:44,914 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:44,915 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:44,916 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:45,447 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:46,447 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:46,489 DEBUG   SenderThread:9504 [sender.py:send():382] send: stats
+2024-08-03 19:19:46,490 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:19:46,525 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-03 19:19:46,526 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: stop_status
+2024-08-03 19:19:46,527 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-03 19:19:47,851 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:47,852 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:47,852 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:47,853 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:48,449 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:50,450 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:50,810 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:50,814 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:50,814 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:50,815 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:51,451 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:51,815 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:19:52,452 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:53,748 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:53,750 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:53,750 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:53,752 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:54,454 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:56,455 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:56,707 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:56,709 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:56,709 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:56,711 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:19:57,456 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:19:57,712 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:19:58,456 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:19:59,749 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:19:59,752 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:19:59,752 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:19:59,754 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:00,458 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:01,525 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-03 19:20:01,526 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: stop_status
+2024-08-03 19:20:01,527 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-03 19:20:02,459 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:02,687 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:02,689 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:02,689 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:02,690 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:02,729 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:20:03,460 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:04,460 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:05,622 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:05,624 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:05,625 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:05,626 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:06,462 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:08,463 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:08,560 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:08,563 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:08,563 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:08,563 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:20:08,564 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:09,464 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:10,464 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:11,503 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:11,505 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:11,506 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:11,507 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:12,466 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:12,466 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:14,443 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:14,468 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:14,469 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:14,469 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:20:14,470 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:15,470 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:16,471 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:16,490 DEBUG   SenderThread:9504 [sender.py:send():382] send: stats
+2024-08-03 19:20:16,525 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-03 19:20:16,526 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: stop_status
+2024-08-03 19:20:16,527 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-03 19:20:17,382 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:17,384 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:17,384 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:17,385 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:17,471 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:18,472 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:20,336 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:20,338 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:20,339 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:20,339 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:20:20,340 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:20,473 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:22,474 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:23,282 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:23,284 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:23,285 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:23,286 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:23,475 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:24,476 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:26,221 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:26,223 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:26,224 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:26,224 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:20:26,225 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:26,477 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:28,478 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:29,171 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:29,173 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:29,174 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:29,175 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:29,479 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:30,480 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:31,525 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-03 19:20:31,526 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: stop_status
+2024-08-03 19:20:31,527 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-03 19:20:31,709 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:20:32,112 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:32,114 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:32,114 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:32,115 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:32,482 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:34,483 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:35,058 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:35,060 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:35,061 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:35,062 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:35,484 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:36,485 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:37,062 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:20:37,999 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:38,001 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:38,002 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:38,003 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:38,486 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:40,487 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:41,029 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:41,031 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:41,032 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:41,033 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:41,488 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:42,489 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:43,034 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:20:43,967 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:43,971 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:43,971 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:43,972 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:44,490 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:46,491 DEBUG   SenderThread:9504 [sender.py:send():382] send: stats
+2024-08-03 19:20:46,492 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:46,525 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-03 19:20:46,526 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: stop_status
+2024-08-03 19:20:46,527 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-03 19:20:46,904 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-03 19:20:46,905 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:46,906 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:46,907 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:47,493 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:47,496 DEBUG   SenderThread:9504 [sender.py:send():382] send: exit
+2024-08-03 19:20:47,496 INFO    SenderThread:9504 [sender.py:send_exit():589] handling exit code: 255
+2024-08-03 19:20:47,496 INFO    SenderThread:9504 [sender.py:send_exit():591] handling runtime: 151
+2024-08-03 19:20:47,497 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:47,497 INFO    SenderThread:9504 [sender.py:send_exit():597] send defer
+2024-08-03 19:20:47,497 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:47,497 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 0
+2024-08-03 19:20:47,497 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:47,497 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 0
+2024-08-03 19:20:47,497 INFO    SenderThread:9504 [sender.py:transition_state():617] send defer: 1
+2024-08-03 19:20:47,498 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:47,498 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 1
+2024-08-03 19:20:47,498 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:47,498 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 1
+2024-08-03 19:20:47,498 INFO    SenderThread:9504 [sender.py:transition_state():617] send defer: 2
+2024-08-03 19:20:47,498 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:47,498 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 2
+2024-08-03 19:20:47,498 INFO    HandlerThread:9504 [system_monitor.py:finish():203] Stopping system monitor
+2024-08-03 19:20:47,498 INFO    HandlerThread:9504 [interfaces.py:finish():202] Joined cpu monitor
+2024-08-03 19:20:47,498 DEBUG   SystemMonitor:9504 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-08-03 19:20:47,499 INFO    HandlerThread:9504 [interfaces.py:finish():202] Joined disk monitor
+2024-08-03 19:20:47,499 DEBUG   SystemMonitor:9504 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-08-03 19:20:47,532 INFO    HandlerThread:9504 [interfaces.py:finish():202] Joined gpu monitor
+2024-08-03 19:20:47,532 INFO    HandlerThread:9504 [interfaces.py:finish():202] Joined memory monitor
+2024-08-03 19:20:47,532 INFO    HandlerThread:9504 [interfaces.py:finish():202] Joined network monitor
+2024-08-03 19:20:47,533 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:47,533 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 2
+2024-08-03 19:20:47,533 INFO    SenderThread:9504 [sender.py:transition_state():617] send defer: 3
+2024-08-03 19:20:47,533 DEBUG   SenderThread:9504 [sender.py:send():382] send: stats
+2024-08-03 19:20:47,533 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:47,533 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 3
+2024-08-03 19:20:47,536 DEBUG   SenderThread:9504 [sender.py:send():382] send: history
+2024-08-03 19:20:47,537 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: summary_record
+2024-08-03 19:20:47,537 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:47,538 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:47,538 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 3
+2024-08-03 19:20:47,538 INFO    SenderThread:9504 [sender.py:transition_state():617] send defer: 4
+2024-08-03 19:20:47,538 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:47,538 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 4
+2024-08-03 19:20:47,538 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:47,538 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 4
+2024-08-03 19:20:47,538 INFO    SenderThread:9504 [sender.py:transition_state():617] send defer: 5
+2024-08-03 19:20:47,538 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:47,538 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 5
+2024-08-03 19:20:47,539 DEBUG   SenderThread:9504 [sender.py:send():382] send: summary
+2024-08-03 19:20:47,540 INFO    SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-03 19:20:47,540 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:47,540 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 5
+2024-08-03 19:20:47,540 INFO    SenderThread:9504 [sender.py:transition_state():617] send defer: 6
+2024-08-03 19:20:47,541 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:47,541 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 6
+2024-08-03 19:20:47,541 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:47,541 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 6
+2024-08-03 19:20:47,541 INFO    SenderThread:9504 [sender.py:transition_state():617] send defer: 7
+2024-08-03 19:20:47,541 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:20:47,541 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:47,541 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 7
+2024-08-03 19:20:47,541 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:47,541 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 7
+2024-08-03 19:20:47,875 INFO    SenderThread:9504 [sender.py:transition_state():617] send defer: 8
+2024-08-03 19:20:47,875 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:47,875 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 8
+2024-08-03 19:20:47,875 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:47,875 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 8
+2024-08-03 19:20:47,875 INFO    SenderThread:9504 [job_builder.py:build():296] Attempting to build job artifact
+2024-08-03 19:20:47,876 INFO    SenderThread:9504 [job_builder.py:_get_source_type():426] is repo sourced job
+2024-08-03 19:20:47,890 INFO    SenderThread:9504 [job_builder.py:build():402] adding wandb-job metadata file
+2024-08-03 19:20:47,898 INFO    SenderThread:9504 [sender.py:transition_state():617] send defer: 9
+2024-08-03 19:20:47,899 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:47,899 DEBUG   SenderThread:9504 [sender.py:send():382] send: artifact
+2024-08-03 19:20:47,899 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 9
+2024-08-03 19:20:48,494 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:48,494 INFO    Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:48,496 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-03 19:20:49,057 INFO    wandb-upload_1:9504 [upload_job.py:push():86] Skipped uploading /singularity_home/.local/share/wandb/artifacts/staging/tmp1gtfugn3
+2024-08-03 19:20:49,437 INFO    wandb-upload_0:9504 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmp8rydwr53
+2024-08-03 19:20:50,875 INFO    SenderThread:9504 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA5MTk4ODAyMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA5MTk2NTkzOA==', 'versionIndex': 1}}}
+2024-08-03 19:20:50,876 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:50,876 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 9
+2024-08-03 19:20:50,876 INFO    SenderThread:9504 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-08-03 19:20:51,495 INFO    SenderThread:9504 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240803_191815-jdwps0z3/files
+2024-08-03 19:20:51,495 INFO    SenderThread:9504 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191815-jdwps0z3/files/requirements.txt requirements.txt
+2024-08-03 19:20:51,495 INFO    SenderThread:9504 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191815-jdwps0z3/files/config.yaml config.yaml
+2024-08-03 19:20:51,497 INFO    SenderThread:9504 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-metadata.json wandb-metadata.json
+2024-08-03 19:20:51,497 INFO    SenderThread:9504 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json wandb-summary.json
+2024-08-03 19:20:51,498 INFO    SenderThread:9504 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log output.log
+2024-08-03 19:20:51,500 INFO    SenderThread:9504 [sender.py:transition_state():617] send defer: 10
+2024-08-03 19:20:51,500 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: poll_exit
+2024-08-03 19:20:51,502 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:51,502 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 10
+2024-08-03 19:20:51,502 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:51,502 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 10
+2024-08-03 19:20:51,502 INFO    SenderThread:9504 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-03 19:20:51,911 INFO    wandb-upload_0:9504 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191815-jdwps0z3/files/config.yaml
+2024-08-03 19:20:51,994 INFO    wandb-upload_1:9504 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191815-jdwps0z3/files/requirements.txt
+2024-08-03 19:20:52,073 INFO    wandb-upload_2:9504 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json
+2024-08-03 19:20:52,122 INFO    wandb-upload_3:9504 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191815-jdwps0z3/files/output.log
+2024-08-03 19:20:52,322 INFO    Thread-11 (_thread_body):9504 [sender.py:transition_state():617] send defer: 11
+2024-08-03 19:20:52,322 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:52,322 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 11
+2024-08-03 19:20:52,323 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:52,323 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 11
+2024-08-03 19:20:52,323 INFO    SenderThread:9504 [file_pusher.py:join():178] waiting for file pusher
+2024-08-03 19:20:52,323 INFO    SenderThread:9504 [sender.py:transition_state():617] send defer: 12
+2024-08-03 19:20:52,323 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:52,323 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 12
+2024-08-03 19:20:52,323 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:52,323 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 12
+2024-08-03 19:20:52,323 INFO    SenderThread:9504 [file_stream.py:finish():595] file stream finish called
+2024-08-03 19:20:52,498 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-03 19:20:52,502 INFO    SenderThread:9504 [file_stream.py:finish():599] file stream finish is done
+2024-08-03 19:20:52,502 INFO    SenderThread:9504 [sender.py:transition_state():617] send defer: 13
+2024-08-03 19:20:52,502 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: poll_exit
+2024-08-03 19:20:52,502 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:52,502 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 13
+2024-08-03 19:20:52,503 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:52,503 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 13
+2024-08-03 19:20:52,503 INFO    SenderThread:9504 [sender.py:transition_state():617] send defer: 14
+2024-08-03 19:20:52,503 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer
+2024-08-03 19:20:52,503 DEBUG   SenderThread:9504 [sender.py:send():382] send: final
+2024-08-03 19:20:52,503 INFO    HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 14
+2024-08-03 19:20:52,503 DEBUG   SenderThread:9504 [sender.py:send():382] send: footer
+2024-08-03 19:20:52,503 DEBUG   SenderThread:9504 [sender.py:send_request():409] send_request: defer
+2024-08-03 19:20:52,503 INFO    SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 14
+2024-08-03 19:20:56,504 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:21:01,505 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:21:06,505 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:21:11,506 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:21:16,506 DEBUG   HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report
+2024-08-03 19:21:21,065 WARNING StreamThr :9504 [internal.py:is_dead():414] Internal process exiting, parent pid 9433 disappeared
+2024-08-03 19:21:21,065 ERROR   StreamThr :9504 [internal.py:wandb_internal():152] Internal process shutdown.
+2024-08-03 19:21:21,507 INFO    SenderThread:9504 [sender.py:finish():1572] shutting down sender
+2024-08-03 19:21:21,507 INFO    SenderThread:9504 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-03 19:21:21,507 INFO    SenderThread:9504 [file_pusher.py:join():178] waiting for file pusher
+2024-08-03 19:21:21,507 INFO    HandlerThread:9504 [handler.py:finish():869] shutting down handler
+2024-08-03 19:21:21,507 INFO    WriterThread:9504 [datastore.py:close():296] close: /project/wandb/run-20240803_191815-jdwps0z3/run-jdwps0z3.wandb

wandb/run-20240803_191815-jdwps0z3/logs/debug.log

@@ -0,0 +1,29 @@
+2024-08-03 19:18:15,865 INFO    MainThread:9433 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
+2024-08-03 19:18:15,866 INFO    MainThread:9433 [wandb_setup.py:_flush():76] Configure stats pid to 9433
+2024-08-03 19:18:15,866 INFO    MainThread:9433 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings
+2024-08-03 19:18:15,866 INFO    MainThread:9433 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings
+2024-08-03 19:18:15,866 INFO    MainThread:9433 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tuny llama sample'}
+2024-08-03 19:18:15,866 INFO    MainThread:9433 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-08-03 19:18:15,866 INFO    MainThread:9433 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'}
+2024-08-03 19:18:15,866 INFO    MainThread:9433 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240803_191815-jdwps0z3/logs/debug.log
+2024-08-03 19:18:15,866 INFO    MainThread:9433 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240803_191815-jdwps0z3/logs/debug-internal.log
+2024-08-03 19:18:15,866 INFO    MainThread:9433 [wandb_init.py:init():566] calling init triggers
+2024-08-03 19:18:15,866 INFO    MainThread:9433 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
+config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 512, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-mistral-sample2_train_2024-08-03-19:18:05', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-mistral-sample2', 'save': '/work/llm_recipes/models/tiny-mistral-sample2', 'base_model': '/share/pretrained_lm/custom/tiny-mistral', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 1600, 'micro_batch_size': 40, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-mistral-sample2', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32768, 'gradient_accumulation_steps': 40}
+2024-08-03 19:18:15,866 INFO    MainThread:9433 [wandb_init.py:init():616] starting backend
+2024-08-03 19:18:15,866 INFO    MainThread:9433 [wandb_init.py:init():620] setting up manager
+2024-08-03 19:18:15,871 INFO    MainThread:9433 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-08-03 19:18:15,872 INFO    MainThread:9433 [wandb_init.py:init():628] backend started and connected
+2024-08-03 19:18:15,877 INFO    MainThread:9433 [wandb_init.py:init():720] updated telemetry
+2024-08-03 19:18:15,887 INFO    MainThread:9433 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-08-03 19:18:16,387 INFO    MainThread:9433 [wandb_run.py:_on_init():2262] communicating current version
+2024-08-03 19:18:16,470 INFO    MainThread:9433 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-08-03 19:18:16,470 INFO    MainThread:9433 [wandb_init.py:init():804] starting run threads in backend
+2024-08-03 19:18:16,525 INFO    MainThread:9433 [wandb_run.py:_console_start():2241] atexit reg
+2024-08-03 19:18:16,525 INFO    MainThread:9433 [wandb_run.py:_redirect():2096] redirect: wrap_raw
+2024-08-03 19:18:16,525 INFO    MainThread:9433 [wandb_run.py:_redirect():2161] Wrapping output streams.
+2024-08-03 19:18:16,525 INFO    MainThread:9433 [wandb_run.py:_redirect():2186] Redirects installed.
+2024-08-03 19:18:16,526 INFO    MainThread:9433 [wandb_init.py:init():847] run started, returning control to user process
+2024-08-03 19:18:17,990 INFO    MainThread:9433 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'silu', 'hidden_size': 256, 'model_type': 'mistral', 'max_position_embeddings': 512, 'num_attention_heads': 4, 'num_hidden_layers': 4, 'model_architecture': 'MistralForCausalLM'}
+2024-08-03 19:18:17,990 INFO    MainThread:9433 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1}

wandb/run-20240803_192355-n3hnzq4n/files/config.yaml

@@ -0,0 +1,335 @@
+wandb_version: 1
+
+sharding_strategy:
+  desc: null
+  value: FULL_SHARD
+checkpoint_type:
+  desc: null
+  value: LOCAL_STATE_DICT
+fsdp_activation_checkpointing:
+  desc: null
+  value: true
+fsdp_cpu_offload:
+  desc: null
+  value: false
+low_cpu_fsdp:
+  desc: null
+  value: false
+no_meta_device:
+  desc: null
+  value: false
+data_path:
+  desc: null
+  value: null
+split:
+  desc: null
+  value: 969, 30, 1
+train_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+valid_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+test_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+data_cache_path:
+  desc: null
+  value: null
+vocab_size:
+  desc: null
+  value: null
+vocab_file:
+  desc: null
+  value: null
+merge_file:
+  desc: null
+  value: null
+seq_length:
+  desc: null
+  value: 512
+num_workers:
+  desc: null
+  value: 2
+tokenizer_type:
+  desc: null
+  value: Llama2Tokenizer
+tokenizer_model:
+  desc: null
+  value: /share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3
+reset_position_ids:
+  desc: null
+  value: false
+reset_attention_mask:
+  desc: null
+  value: false
+eod_mask_loss:
+  desc: null
+  value: false
+retro_return_doc_ids:
+  desc: null
+  value: false
+short_seq_prob:
+  desc: null
+  value: 0.1
+vocab_extra_ids:
+  desc: null
+  value: 0
+seed:
+  desc: null
+  value: 1234
+use_mpi:
+  desc: null
+  value: false
+wandb_entity:
+  desc: null
+  value: iwakawa-koichi-q5-tohoku-nlp6723
+wandb_name:
+  desc: null
+  value: tiny-mistral-sample2_train_2024-08-03-19:23:42
+wandb_project:
+  desc: null
+  value: llm_tutorial
+quantization:
+  desc: null
+  value: false
+use_freeze_layers:
+  desc: null
+  value: false
+freeze_layers:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+mixed_precision:
+  desc: null
+  value: true
+param_dtype:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: /work/llm_recipes/models/tiny-mistral-sample2
+save:
+  desc: null
+  value: /work/llm_recipes/models/tiny-mistral-sample2
+base_model:
+  desc: null
+  value: /share/pretrained_lm/custom/tiny-mistral
+use_better_transformer:
+  desc: null
+  value: false
+grad_clip_norm:
+  desc: null
+  value: 1.0
+eval_interval:
+  desc: null
+  value: 200
+save_interval:
+  desc: null
+  value: 200
+eval_iters:
+  desc: null
+  value: 10
+optimizer:
+  desc: null
+  value: adam
+lr:
+  desc: null
+  value: 2.0e-05
+lr_decay_style:
+  desc: null
+  value: cosine
+lr_decay_iters:
+  desc: null
+  value: 20000
+lr_warmup_iters:
+  desc: null
+  value: 500
+min_lr:
+  desc: null
+  value: 1.0e-06
+train_iters:
+  desc: null
+  value: 20000
+train_samples:
+  desc: null
+  value: null
+global_batch_size:
+  desc: null
+  value: 1600
+micro_batch_size:
+  desc: null
+  value: 40
+make_vocab_size_divisible_by:
+  desc: null
+  value: 128
+sliding_window_size:
+  desc: null
+  value: 4096
+skip_batch:
+  desc: null
+  value: null
+no_save_optimizer_state:
+  desc: null
+  value: false
+continual_pretraining:
+  desc: null
+  value: false
+instruction_tuning:
+  desc: null
+  value: false
+direct_preference_optimization:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.1
+hidden_dropout:
+  desc: null
+  value: 0.1
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_eps:
+  desc: null
+  value: 1.0e-06
+hf_transformer_model_dir:
+  desc: null
+  value: null
+instruction_train_data_path:
+  desc: null
+  value: null
+instruction_valid_data_path:
+  desc: null
+  value: null
+epoch:
+  desc: null
+  value: null
+instruction_dataset_size:
+  desc: null
+  value: null
+save_sampler_state:
+  desc: null
+  value: false
+label_smoothing:
+  desc: null
+  value: 0.0
+save_n_checkpoints:
+  desc: null
+  value: 10
+hf_repo_id:
+  desc: null
+  value: koichi12/tiny-mistral-sample2
+create_public_hf_repo:
+  desc: null
+  value: false
+upload_all_checkpoints_to_hf:
+  desc: null
+  value: false
+hf_upload_retry_limit:
+  desc: null
+  value: 2
+exit_duration_in_mins:
+  desc: null
+  value: null
+source_key:
+  desc: null
+  value: null
+target_key:
+  desc: null
+  value: null
+attn_implementation:
+  desc: null
+  value: flash_attention_2
+efficient_instruction_tuning:
+  desc: null
+  value: false
+remove_padding_masking:
+  desc: null
+  value: false
+save_start_iter:
+  desc: null
+  value: null
+rank:
+  desc: null
+  value: 0
+world_size:
+  desc: null
+  value: 1
+padded_vocab_size:
+  desc: null
+  value: 32768
+gradient_accumulation_steps:
+  desc: null
+  value: 40
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.43.3
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1722680635.371313
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      2:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.43.3
+      8:
+      - 5
+      13: linux-x86_64
+activation_function:
+  desc: null
+  value: silu
+hidden_size:
+  desc: null
+  value: 256
+model_type:
+  desc: null
+  value: mistral
+max_position_embeddings:
+  desc: null
+  value: 512
+num_attention_heads:
+  desc: null
+  value: 4
+num_hidden_layers:
+  desc: null
+  value: 4
+model_architecture:
+  desc: null
+  value: MistralForCausalLM

wandb/run-20240803_192355-n3hnzq4n/files/requirements.txt

@@ -0,0 +1,271 @@
+absl-py==2.1.0
+accelerate==0.33.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+apex==0.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+beautifulsoup4==4.12.3
+bleach==6.1.0
+blis==0.7.11
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+cmake==3.28.1
+colorama==0.4.6
+comm==0.2.1
+confection==0.1.4
+contourpy==1.2.0
+cubinlinker==0.3.0+2.g405ac64
+cuda-python==12.3.0rc4+9.gdb8c48a.dirty
+cudf==23.12.0
+cugraph-dgl==23.12.0
+cugraph-service-client==23.12.0
+cugraph-service-server==23.12.0
+cugraph==23.12.0
+cuml==23.12.0
+cupy-cuda12x==12.3.0
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.8
+dask-cuda==23.12.0
+dask-cudf==23.12.0
+dask==2023.11.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+distributed==2023.11.0
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+einops==0.7.0
+exceptiongroup==1.2.0
+execnet==2.0.2
+executing==2.0.1
+expecttest==0.1.3
+fastjsonschema==2.19.1
+fastrlock==0.8.2
+filelock==3.13.1
+flash-attn==2.4.2
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2023.12.2
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.27.0
+graphsurgeon==0.4.6
+grpcio==1.60.1
+huggingface-hub==0.24.5
+hypothesis==5.35.1
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+intel-openmp==2021.4.0
+ipadic==1.0.0
+ipykernel==6.29.2
+ipython-genutils==0.2.0
+ipython==8.21.0
+jedi==0.19.1
+jinja2==3.1.3
+joblib==1.3.2
+json5==0.9.14
+jsonnet==0.19.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.21.1
+jupyter-client==8.6.0
+jupyter-core==5.7.1
+jupyter-tensorboard==0.2.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==1.2.0
+jupyterlab==2.3.2
+jupytext==1.16.1
+kiwisolver==1.4.5
+langcodes==3.3.0
+lazy-loader==0.3
+librosa==0.10.1
+llvmlite==0.40.1
+locket==1.0.0
+logzero==1.7.0
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown==3.5.2
+markupsafe==2.1.4
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mecab-python3==1.0.6
+mistune==3.0.2
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mkl==2021.1.1
+mock==5.1.0
+more-itertools==9.1.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+murmurhash==1.0.10
+nbclient==0.9.0
+nbconvert==7.16.0
+nbformat==5.9.2
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.4.10
+numba==0.57.1+1.g1ff679645
+numpy==1.24.4
+nvfuser==0.1.4a0+d0bb811
+nvidia-dali-cuda120==1.34.0
+nvidia-pyindex==1.0.9
+nvtx==0.2.5
+oauthlib==3.2.2
+onnx==1.15.0rc2
+opencv==4.7.0
+optree==0.10.0
+packaging==23.2
+pandas==1.5.3
+pandocfilters==1.5.1
+parso==0.8.3
+partd==1.4.1
+peft==0.11.1
+pexpect==4.9.0
+pillow==10.2.0
+pip==24.0
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+polygraphy==0.49.4
+pooch==1.8.0
+portalocker==2.10.1
+preshed==3.0.9
+prettytable==3.9.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.24.4
+psutil==5.9.4
+ptxcompiler==0.8.1+2.g0d406d6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==14.0.1.dev0+gba5374836.d20240125
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pybind11-global==2.11.1
+pybind11==2.11.1
+pycocotools==2.0+nv0.8.0
+pycparser==2.21
+pydantic-core==2.16.2
+pydantic==2.6.1
+pygments==2.17.2
+pylibcugraph==23.12.0
+pylibcugraphops==23.12.0
+pylibraft==23.12.0
+pynvml==11.4.1
+pyparsing==3.1.1
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==13.0
+pytest-shard==0.1.2
+pytest-xdist==3.5.0
+pytest==8.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-hostlist==1.23.0
+pytorch-quantization==2.1.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.2
+raft-dask==23.12.0
+rapids-dask-dependency==23.12.1
+referencing==0.33.0
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests==2.31.0
+rich==13.7.0
+rmm==23.12.0
+rpds-py==0.17.1
+rsa==4.9
+sacrebleu==2.4.0
+safetensors==0.4.3
+scikit-learn==1.2.0
+scipy==1.12.0
+send2trash==1.8.2
+sentencepiece==0.1.99
+sentry-sdk==2.12.0
+setproctitle==1.3.3
+setuptools==68.2.2
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sphinx-glpi-theme==0.6
+srsly==2.4.8
+stack-data==0.6.3
+sympy==1.12
+tabulate==0.9.0
+tbb==2021.11.0
+tblib==3.0.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.9.0
+tensorrt==8.6.3
+terminado==0.18.0
+termplotlib==0.3.9
+thinc==8.2.3
+threadpoolctl==3.2.0
+thriftpy2==0.4.17
+tinycss2==1.2.1
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.1
+torch-tensorrt==2.3.0a0
+torch==2.3.0a0+ebedce2
+torchdata==0.7.1a0
+torchtext==0.17.0a0
+torchvision==0.18.0a0
+tornado==6.4
+tqdm==4.66.1
+traitlets==5.9.0
+transformer-engine==1.3.0+5b90b7f
+transformers==4.43.3
+treelite-runtime==3.9.1
+treelite==3.9.1
+triton==2.2.0+e28a256
+typer==0.9.0
+types-dataclasses==0.6.6
+typing-extensions==4.9.0
+ucx-py==0.35.0
+uff==0.6.9
+ujson==5.8.0
+urllib3==1.26.18
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.3.4
+webencodings==0.5.1
+werkzeug==3.0.1
+wheel==0.42.0
+xdoctest==1.0.2
+xgboost==1.7.6
+yarl==1.9.4
+zict==3.0.0
+zipp==3.17.0

wandb/run-20240803_192355-n3hnzq4n/files/wandb-metadata.json

@@ -0,0 +1,215 @@
+{
+    "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-08-03T10:23:55.999895",
+    "startedAt": "2024-08-03T10:23:55.358106",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--seq-length",
+        "512",
+        "--sliding-window-size",
+        "4096",
+        "--micro-batch-size",
+        "40",
+        "--global-batch-size",
+        "1600",
+        "--train-iters",
+        "20000",
+        "--tokenizer-type",
+        "Llama2Tokenizer",
+        "--tokenizer-model",
+        "/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3",
+        "--train-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--valid-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--test-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--lr",
+        "2e-5",
+        "--min-lr",
+        "1e-6",
+        "--lr-decay-style",
+        "cosine",
+        "--lr-warmup-iters",
+        "500",
+        "--lr-decay-iters",
+        "20000",
+        "--weight-decay",
+        "0.1",
+        "--grad-clip-norm",
+        "1.0",
+        "--optimizer",
+        "adam",
+        "--adam-beta1",
+        "0.9",
+        "--adam-beta2",
+        "0.95",
+        "--adam-eps",
+        "1e-6",
+        "--save-interval",
+        "200",
+        "--eval-interval",
+        "200",
+        "--eval-iters",
+        "10",
+        "--bf16",
+        "--mixed-precision",
+        "--base-model",
+        "/share/pretrained_lm/custom/tiny-mistral",
+        "--save",
+        "/work/llm_recipes/models/tiny-mistral-sample2",
+        "--load",
+        "/work/llm_recipes/models/tiny-mistral-sample2",
+        "--fsdp-activation-checkpointing",
+        "--sharding-strategy",
+        "FULL_SHARD",
+        "--checkpoint-type",
+        "LOCAL_STATE_DICT",
+        "--save-n-checkpoints",
+        "10",
+        "--hf-upload-retry-limit",
+        "2",
+        "--hf-repo-id",
+        "koichi12/tiny-mistral-sample2",
+        "--wandb-entity",
+        "iwakawa-koichi-q5-tohoku-nlp6723",
+        "--wandb-project",
+        "llm_tutorial",
+        "--wandb-name",
+        "tiny-mistral-sample2_train_2024-08-03-19:23:42"
+    ],
+    "state": "running",
+    "program": "/project/examples/finetuning.py",
+    "codePathLocal": "examples/finetuning.py",
+    "codePath": "examples/finetuning.py",
+    "git": {
+        "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git",
+        "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea"
+    },
+    "email": null,
+    "root": "/project",
+    "host": "gpu-koiwa-00",
+    "username": "koiwa",
+    "executable": "/usr/bin/python",
+    "cpu_count": 18,
+    "cpu_count_logical": 18,
+    "cpu_freq": {
+        "current": 2400.034,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 0.0625,
+            "used": 1.1444091796875e-05
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 56.48782730102539
+    }
+}

wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json

@@ -0,0 +1 @@
+{"training/loss": 8.676288604736328, "training/perplexity": 5862.24904336478, "utils/batch_size": 40, "utils/global_batch_size": 1600, "utils/seq_len": 513, "utils/gradient_accumulation_steps": 40, "utils/iteration": 2828, "optimizer/lr": 1.933962231874466e-05, "optimizer/variance_l2": 0.014160344368509084, "optimizer/variance_sqrt_l2": 0.9983195251588314, "optimizer/momentum_l2": 0.9847836932741917, "optimizer/weight_l2": 101.93656115447489, "optimizer/variance_l1": 1.0000762939453125, "optimizer/variance_sqrt_l1": 530.25, "optimizer/momentum_l1": 418.75, "optimizer/weight_l1": 333248.0, "optimizer/variance_abs_max": 0.00130462646484375, "optimizer/variance_sqrt_abs_max": 0.0361328125, "optimizer/momentum_abs_max": 0.03662109375, "optimizer/weight_abs_max": 1.0, "stats/1_iteration_time": 2.9201389469999413, "stats/tokens_per_sec": 281082.5152149914, "stats/tokens_per_sec_per_gpu": 281082.5152149914, "stats/tflops": 19.898159556447013, "_timestamp": 1722688970.2608888, "_runtime": 8334.889575719833, "_step": 2828, "evaluation/val_loss": 8.68109130859375, "evaluation/val_ppl": 5890.47216796875, "_wandb": {"runtime": 8336}}

wandb/run-20240803_192355-n3hnzq4n/logs/debug.log

@@ -0,0 +1,29 @@
+2024-08-03 19:23:55,363 INFO    MainThread:10080 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
+2024-08-03 19:23:55,364 INFO    MainThread:10080 [wandb_setup.py:_flush():76] Configure stats pid to 10080
+2024-08-03 19:23:55,364 INFO    MainThread:10080 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings
+2024-08-03 19:23:55,364 INFO    MainThread:10080 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings
+2024-08-03 19:23:55,364 INFO    MainThread:10080 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tuny llama sample'}
+2024-08-03 19:23:55,364 INFO    MainThread:10080 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-08-03 19:23:55,364 INFO    MainThread:10080 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'}
+2024-08-03 19:23:55,364 INFO    MainThread:10080 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240803_192355-n3hnzq4n/logs/debug.log
+2024-08-03 19:23:55,364 INFO    MainThread:10080 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240803_192355-n3hnzq4n/logs/debug-internal.log
+2024-08-03 19:23:55,364 INFO    MainThread:10080 [wandb_init.py:init():566] calling init triggers
+2024-08-03 19:23:55,364 INFO    MainThread:10080 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
+config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 512, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-mistral-sample2_train_2024-08-03-19:23:42', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-mistral-sample2', 'save': '/work/llm_recipes/models/tiny-mistral-sample2', 'base_model': '/share/pretrained_lm/custom/tiny-mistral', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 1600, 'micro_batch_size': 40, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-mistral-sample2', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32768, 'gradient_accumulation_steps': 40}
+2024-08-03 19:23:55,364 INFO    MainThread:10080 [wandb_init.py:init():616] starting backend
+2024-08-03 19:23:55,364 INFO    MainThread:10080 [wandb_init.py:init():620] setting up manager
+2024-08-03 19:23:55,369 INFO    MainThread:10080 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-08-03 19:23:55,371 INFO    MainThread:10080 [wandb_init.py:init():628] backend started and connected
+2024-08-03 19:23:55,375 INFO    MainThread:10080 [wandb_init.py:init():720] updated telemetry
+2024-08-03 19:23:55,403 INFO    MainThread:10080 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-08-03 19:23:55,888 INFO    MainThread:10080 [wandb_run.py:_on_init():2262] communicating current version
+2024-08-03 19:23:55,974 INFO    MainThread:10080 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-08-03 19:23:55,974 INFO    MainThread:10080 [wandb_init.py:init():804] starting run threads in backend
+2024-08-03 19:23:56,034 INFO    MainThread:10080 [wandb_run.py:_console_start():2241] atexit reg
+2024-08-03 19:23:56,034 INFO    MainThread:10080 [wandb_run.py:_redirect():2096] redirect: wrap_raw
+2024-08-03 19:23:56,034 INFO    MainThread:10080 [wandb_run.py:_redirect():2161] Wrapping output streams.
+2024-08-03 19:23:56,035 INFO    MainThread:10080 [wandb_run.py:_redirect():2186] Redirects installed.
+2024-08-03 19:23:56,035 INFO    MainThread:10080 [wandb_init.py:init():847] run started, returning control to user process
+2024-08-03 19:23:57,141 INFO    MainThread:10080 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'silu', 'hidden_size': 256, 'model_type': 'mistral', 'max_position_embeddings': 512, 'num_attention_heads': 4, 'num_hidden_layers': 4, 'model_architecture': 'MistralForCausalLM'}
+2024-08-03 19:23:57,141 INFO    MainThread:10080 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1}

wandb/run-20240812_063027-j1htzx7q/files/config.yaml

@@ -0,0 +1,335 @@
+wandb_version: 1
+
+sharding_strategy:
+  desc: null
+  value: FULL_SHARD
+checkpoint_type:
+  desc: null
+  value: LOCAL_STATE_DICT
+fsdp_activation_checkpointing:
+  desc: null
+  value: true
+fsdp_cpu_offload:
+  desc: null
+  value: false
+low_cpu_fsdp:
+  desc: null
+  value: false
+no_meta_device:
+  desc: null
+  value: false
+data_path:
+  desc: null
+  value: null
+split:
+  desc: null
+  value: 969, 30, 1
+train_data_path:
+  desc: null
+  value:
+  - '235289369'
+  - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document
+valid_data_path:
+  desc: null
+  value:
+  - '235289369'
+  - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document
+test_data_path:
+  desc: null
+  value:
+  - '235289369'
+  - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document
+data_cache_path:
+  desc: null
+  value: null
+vocab_size:
+  desc: null
+  value: null
+vocab_file:
+  desc: null
+  value: null
+merge_file:
+  desc: null
+  value: null
+seq_length:
+  desc: null
+  value: 4096
+num_workers:
+  desc: null
+  value: 2
+tokenizer_type:
+  desc: null
+  value: HFPreTrainedTokenizer
+tokenizer_model:
+  desc: null
+  value: /share/pretrained_lm/google/gemma-2-2b
+reset_position_ids:
+  desc: null
+  value: false
+reset_attention_mask:
+  desc: null
+  value: false
+eod_mask_loss:
+  desc: null
+  value: false
+retro_return_doc_ids:
+  desc: null
+  value: false
+short_seq_prob:
+  desc: null
+  value: 0.1
+vocab_extra_ids:
+  desc: null
+  value: 0
+seed:
+  desc: null
+  value: 1234
+use_mpi:
+  desc: null
+  value: false
+wandb_entity:
+  desc: null
+  value: iwakawa-koichi-q5-tohoku-nlp6723
+wandb_name:
+  desc: null
+  value: yans-sample-gemma-2-2b_train_2024-08-12-06:30:12
+wandb_project:
+  desc: null
+  value: llm_tutorial
+quantization:
+  desc: null
+  value: false
+use_freeze_layers:
+  desc: null
+  value: false
+freeze_layers:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+mixed_precision:
+  desc: null
+  value: true
+param_dtype:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: /work/llm_recipes/models/yans-sample-gemma-2-2b
+save:
+  desc: null
+  value: /work/llm_recipes/models/yans-sample-gemma-2-2b
+base_model:
+  desc: null
+  value: /share/pretrained_lm/google/gemma-2-2b
+use_better_transformer:
+  desc: null
+  value: false
+grad_clip_norm:
+  desc: null
+  value: 1.0
+eval_interval:
+  desc: null
+  value: 3
+save_interval:
+  desc: null
+  value: 3
+eval_iters:
+  desc: null
+  value: 10
+optimizer:
+  desc: null
+  value: anyprecision
+lr:
+  desc: null
+  value: 2.0e-05
+lr_decay_style:
+  desc: null
+  value: cosine
+lr_decay_iters:
+  desc: null
+  value: 20000
+lr_warmup_iters:
+  desc: null
+  value: 500
+min_lr:
+  desc: null
+  value: 1.0e-06
+train_iters:
+  desc: null
+  value: 20000
+train_samples:
+  desc: null
+  value: null
+global_batch_size:
+  desc: null
+  value: 320
+micro_batch_size:
+  desc: null
+  value: 1
+make_vocab_size_divisible_by:
+  desc: null
+  value: 128
+sliding_window_size:
+  desc: null
+  value: 4096
+skip_batch:
+  desc: null
+  value: null
+no_save_optimizer_state:
+  desc: null
+  value: false
+continual_pretraining:
+  desc: null
+  value: false
+instruction_tuning:
+  desc: null
+  value: false
+direct_preference_optimization:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.1
+hidden_dropout:
+  desc: null
+  value: 0.1
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_eps:
+  desc: null
+  value: 1.0e-06
+hf_transformer_model_dir:
+  desc: null
+  value: null
+instruction_train_data_path:
+  desc: null
+  value: null
+instruction_valid_data_path:
+  desc: null
+  value: null
+epoch:
+  desc: null
+  value: null
+instruction_dataset_size:
+  desc: null
+  value: null
+save_sampler_state:
+  desc: null
+  value: false
+label_smoothing:
+  desc: null
+  value: 0.0
+save_n_checkpoints:
+  desc: null
+  value: 10
+hf_repo_id:
+  desc: null
+  value: koichi12/yans-sample-gemma-2-2b
+create_public_hf_repo:
+  desc: null
+  value: false
+upload_all_checkpoints_to_hf:
+  desc: null
+  value: false
+hf_upload_retry_limit:
+  desc: null
+  value: 2
+exit_duration_in_mins:
+  desc: null
+  value: null
+source_key:
+  desc: null
+  value: null
+target_key:
+  desc: null
+  value: null
+attn_implementation:
+  desc: null
+  value: flash_attention_2
+efficient_instruction_tuning:
+  desc: null
+  value: false
+remove_padding_masking:
+  desc: null
+  value: false
+save_start_iter:
+  desc: null
+  value: null
+rank:
+  desc: null
+  value: 0
+world_size:
+  desc: null
+  value: 1
+padded_vocab_size:
+  desc: null
+  value: 256000
+gradient_accumulation_steps:
+  desc: null
+  value: 320
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.43.3
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1723411827.601845
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      2:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.43.3
+      8:
+      - 5
+      13: linux-x86_64
+model_architecture:
+  desc: null
+  value: Gemma2ForCausalLM
+activation_function:
+  desc: null
+  value: gelu_pytorch_tanh
+hidden_size:
+  desc: null
+  value: 2304
+model_type:
+  desc: null
+  value: gemma2
+max_position_embeddings:
+  desc: null
+  value: 4096
+num_attention_heads:
+  desc: null
+  value: 8
+num_hidden_layers:
+  desc: null
+  value: 26

wandb/run-20240812_063027-j1htzx7q/files/requirements.txt

@@ -0,0 +1,271 @@
+absl-py==2.1.0
+accelerate==0.33.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+apex==0.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+beautifulsoup4==4.12.3
+bleach==6.1.0
+blis==0.7.11
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+cmake==3.28.1
+colorama==0.4.6
+comm==0.2.1
+confection==0.1.4
+contourpy==1.2.0
+cubinlinker==0.3.0+2.g405ac64
+cuda-python==12.3.0rc4+9.gdb8c48a.dirty
+cudf==23.12.0
+cugraph-dgl==23.12.0
+cugraph-service-client==23.12.0
+cugraph-service-server==23.12.0
+cugraph==23.12.0
+cuml==23.12.0
+cupy-cuda12x==12.3.0
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.8
+dask-cuda==23.12.0
+dask-cudf==23.12.0
+dask==2023.11.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+distributed==2023.11.0
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+einops==0.7.0
+exceptiongroup==1.2.0
+execnet==2.0.2
+executing==2.0.1
+expecttest==0.1.3
+fastjsonschema==2.19.1
+fastrlock==0.8.2
+filelock==3.13.1
+flash-attn==2.4.2
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2023.12.2
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.27.0
+graphsurgeon==0.4.6
+grpcio==1.60.1
+huggingface-hub==0.24.5
+hypothesis==5.35.1
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+intel-openmp==2021.4.0
+ipadic==1.0.0
+ipykernel==6.29.2
+ipython-genutils==0.2.0
+ipython==8.21.0
+jedi==0.19.1
+jinja2==3.1.3
+joblib==1.3.2
+json5==0.9.14
+jsonnet==0.19.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.21.1
+jupyter-client==8.6.0
+jupyter-core==5.7.1
+jupyter-tensorboard==0.2.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==1.2.0
+jupyterlab==2.3.2
+jupytext==1.16.1
+kiwisolver==1.4.5
+langcodes==3.3.0
+lazy-loader==0.3
+librosa==0.10.1
+llvmlite==0.40.1
+locket==1.0.0
+logzero==1.7.0
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown==3.5.2
+markupsafe==2.1.4
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mecab-python3==1.0.6
+mistune==3.0.2
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mkl==2021.1.1
+mock==5.1.0
+more-itertools==9.1.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+murmurhash==1.0.10
+nbclient==0.9.0
+nbconvert==7.16.0
+nbformat==5.9.2
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.4.10
+numba==0.57.1+1.g1ff679645
+numpy==1.24.4
+nvfuser==0.1.4a0+d0bb811
+nvidia-dali-cuda120==1.34.0
+nvidia-pyindex==1.0.9
+nvtx==0.2.5
+oauthlib==3.2.2
+onnx==1.15.0rc2
+opencv==4.7.0
+optree==0.10.0
+packaging==23.2
+pandas==1.5.3
+pandocfilters==1.5.1
+parso==0.8.3
+partd==1.4.1
+peft==0.11.1
+pexpect==4.9.0
+pillow==10.2.0
+pip==24.0
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+polygraphy==0.49.4
+pooch==1.8.0
+portalocker==2.10.1
+preshed==3.0.9
+prettytable==3.9.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.24.4
+psutil==5.9.4
+ptxcompiler==0.8.1+2.g0d406d6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==14.0.1.dev0+gba5374836.d20240125
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pybind11-global==2.11.1
+pybind11==2.11.1
+pycocotools==2.0+nv0.8.0
+pycparser==2.21
+pydantic-core==2.16.2
+pydantic==2.6.1
+pygments==2.17.2
+pylibcugraph==23.12.0
+pylibcugraphops==23.12.0
+pylibraft==23.12.0
+pynvml==11.4.1
+pyparsing==3.1.1
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==13.0
+pytest-shard==0.1.2
+pytest-xdist==3.5.0
+pytest==8.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-hostlist==1.23.0
+pytorch-quantization==2.1.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.2
+raft-dask==23.12.0
+rapids-dask-dependency==23.12.1
+referencing==0.33.0
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests==2.31.0
+rich==13.7.0
+rmm==23.12.0
+rpds-py==0.17.1
+rsa==4.9
+sacrebleu==2.4.0
+safetensors==0.4.3
+scikit-learn==1.2.0
+scipy==1.12.0
+send2trash==1.8.2
+sentencepiece==0.1.99
+sentry-sdk==2.12.0
+setproctitle==1.3.3
+setuptools==68.2.2
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sphinx-glpi-theme==0.6
+srsly==2.4.8
+stack-data==0.6.3
+sympy==1.12
+tabulate==0.9.0
+tbb==2021.11.0
+tblib==3.0.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.9.0
+tensorrt==8.6.3
+terminado==0.18.0
+termplotlib==0.3.9
+thinc==8.2.3
+threadpoolctl==3.2.0
+thriftpy2==0.4.17
+tinycss2==1.2.1
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.1
+torch-tensorrt==2.3.0a0
+torch==2.3.0a0+ebedce2
+torchdata==0.7.1a0
+torchtext==0.17.0a0
+torchvision==0.18.0a0
+tornado==6.4
+tqdm==4.66.1
+traitlets==5.9.0
+transformer-engine==1.3.0+5b90b7f
+transformers==4.43.3
+treelite-runtime==3.9.1
+treelite==3.9.1
+triton==2.2.0+e28a256
+typer==0.9.0
+types-dataclasses==0.6.6
+typing-extensions==4.9.0
+ucx-py==0.35.0
+uff==0.6.9
+ujson==5.8.0
+urllib3==1.26.18
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.3.4
+webencodings==0.5.1
+werkzeug==3.0.1
+wheel==0.42.0
+xdoctest==1.0.2
+xgboost==1.7.6
+yarl==1.9.4
+zict==3.0.0
+zipp==3.17.0

wandb/run-20240812_063027-j1htzx7q/files/wandb-metadata.json

@@ -0,0 +1,215 @@
+{
+    "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-08-11T21:30:28.265073",
+    "startedAt": "2024-08-11T21:30:27.589443",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--seq-length",
+        "4096",
+        "--sliding-window-size",
+        "4096",
+        "--micro-batch-size",
+        "1",
+        "--global-batch-size",
+        "320",
+        "--train-iters",
+        "20000",
+        "--tokenizer-type",
+        "HFPreTrainedTokenizer",
+        "--tokenizer-model",
+        "/share/pretrained_lm/google/gemma-2-2b",
+        "--train-data-path",
+        "235289369",
+        "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document",
+        "--valid-data-path",
+        "235289369",
+        "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document",
+        "--test-data-path",
+        "235289369",
+        "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document",
+        "--lr",
+        "2e-5",
+        "--min-lr",
+        "1e-6",
+        "--lr-decay-style",
+        "cosine",
+        "--lr-warmup-iters",
+        "500",
+        "--lr-decay-iters",
+        "20000",
+        "--weight-decay",
+        "0.1",
+        "--grad-clip-norm",
+        "1.0",
+        "--optimizer",
+        "anyprecision",
+        "--adam-beta1",
+        "0.9",
+        "--adam-beta2",
+        "0.95",
+        "--adam-eps",
+        "1e-6",
+        "--save-interval",
+        "3",
+        "--eval-interval",
+        "3",
+        "--eval-iters",
+        "10",
+        "--bf16",
+        "--mixed-precision",
+        "--base-model",
+        "/share/pretrained_lm/google/gemma-2-2b",
+        "--save",
+        "/work/llm_recipes/models/yans-sample-gemma-2-2b",
+        "--load",
+        "/work/llm_recipes/models/yans-sample-gemma-2-2b",
+        "--fsdp-activation-checkpointing",
+        "--sharding-strategy",
+        "FULL_SHARD",
+        "--checkpoint-type",
+        "LOCAL_STATE_DICT",
+        "--save-n-checkpoints",
+        "10",
+        "--hf-upload-retry-limit",
+        "2",
+        "--hf-repo-id",
+        "koichi12/yans-sample-gemma-2-2b",
+        "--wandb-entity",
+        "iwakawa-koichi-q5-tohoku-nlp6723",
+        "--wandb-project",
+        "llm_tutorial",
+        "--wandb-name",
+        "yans-sample-gemma-2-2b_train_2024-08-12-06:30:12"
+    ],
+    "state": "running",
+    "program": "/project/examples/finetuning.py",
+    "codePathLocal": "examples/finetuning.py",
+    "codePath": "examples/finetuning.py",
+    "git": {
+        "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git",
+        "commit": "6da01327e78c302bc0cfdb335f3ca297e2a19c8c"
+    },
+    "email": null,
+    "root": "/project",
+    "host": "gpu-koiwa-00",
+    "username": "koiwa",
+    "executable": "/usr/bin/python",
+    "cpu_count": 18,
+    "cpu_count_logical": 18,
+    "cpu_freq": {
+        "current": 2400.0429999999997,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 0.0625,
+            "used": 1.1444091796875e-05
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 56.487823486328125
+    }
+}

wandb/run-20240812_063027-j1htzx7q/logs/debug-internal.log

@@ -0,0 +1,261 @@
+2024-08-12 06:30:27,603 INFO    StreamThr :12721 [internal.py:wandb_internal():86] W&B internal server running at pid: 12721, started at: 2024-08-12 06:30:27.602612
+2024-08-12 06:30:27,605 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status
+2024-08-12 06:30:27,607 INFO    WriterThread:12721 [datastore.py:open_for_write():87] open: /project/wandb/run-20240812_063027-j1htzx7q/run-j1htzx7q.wandb
+2024-08-12 06:30:27,608 DEBUG   SenderThread:12721 [sender.py:send():382] send: header
+2024-08-12 06:30:27,640 DEBUG   SenderThread:12721 [sender.py:send():382] send: run
+2024-08-12 06:30:28,148 INFO    SenderThread:12721 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240812_063027-j1htzx7q/files
+2024-08-12 06:30:28,148 INFO    SenderThread:12721 [sender.py:_start_run_threads():1136] run started: j1htzx7q with start time 1723411827.601845
+2024-08-12 06:30:28,154 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: check_version
+2024-08-12 06:30:28,154 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: check_version
+2024-08-12 06:30:28,244 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: run_start
+2024-08-12 06:30:28,251 DEBUG   HandlerThread:12721 [system_info.py:__init__():27] System info init
+2024-08-12 06:30:28,251 DEBUG   HandlerThread:12721 [system_info.py:__init__():42] System info init done
+2024-08-12 06:30:28,251 INFO    HandlerThread:12721 [system_monitor.py:start():194] Starting system monitor
+2024-08-12 06:30:28,251 INFO    SystemMonitor:12721 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-08-12 06:30:28,251 INFO    HandlerThread:12721 [system_monitor.py:probe():214] Collecting system info
+2024-08-12 06:30:28,252 INFO    SystemMonitor:12721 [interfaces.py:start():190] Started cpu monitoring
+2024-08-12 06:30:28,252 INFO    SystemMonitor:12721 [interfaces.py:start():190] Started disk monitoring
+2024-08-12 06:30:28,253 INFO    SystemMonitor:12721 [interfaces.py:start():190] Started gpu monitoring
+2024-08-12 06:30:28,254 INFO    SystemMonitor:12721 [interfaces.py:start():190] Started memory monitoring
+2024-08-12 06:30:28,255 INFO    SystemMonitor:12721 [interfaces.py:start():190] Started network monitoring
+2024-08-12 06:30:28,264 DEBUG   HandlerThread:12721 [system_info.py:probe():151] Probing system
+2024-08-12 06:30:28,267 DEBUG   HandlerThread:12721 [system_info.py:_probe_git():136] Probing git
+2024-08-12 06:30:28,279 DEBUG   HandlerThread:12721 [system_info.py:_probe_git():144] Probing git done
+2024-08-12 06:30:28,279 DEBUG   HandlerThread:12721 [system_info.py:probe():199] Probing system done
+2024-08-12 06:30:28,279 DEBUG   HandlerThread:12721 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-11T21:30:28.265073', 'startedAt': '2024-08-11T21:30:27.589443', 'docker': None, 'cuda': None, 'args': ('--seq-length', '4096', '--sliding-window-size', '4096', '--micro-batch-size', '1', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/google/gemma-2-2b', '--train-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--valid-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--test-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '3', '--eval-interval', '3', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/google/gemma-2-2b', '--save', '/work/llm_recipes/models/yans-sample-gemma-2-2b', '--load', '/work/llm_recipes/models/yans-sample-gemma-2-2b', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-sample-gemma-2-2b', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'yans-sample-gemma-2-2b_train_2024-08-12-06:30:12'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '6da01327e78c302bc0cfdb335f3ca297e2a19c8c'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0429999999997, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.487823486328125}}
+2024-08-12 06:30:28,279 INFO    HandlerThread:12721 [system_monitor.py:probe():224] Finished collecting system info
+2024-08-12 06:30:28,279 INFO    HandlerThread:12721 [system_monitor.py:probe():227] Publishing system info
+2024-08-12 06:30:28,281 INFO    HandlerThread:12721 [system_monitor.py:probe():229] Finished publishing system info
+2024-08-12 06:30:28,287 DEBUG   SenderThread:12721 [sender.py:send():382] send: files
+2024-08-12 06:30:28,287 INFO    SenderThread:12721 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now
+2024-08-12 06:30:28,297 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: python_packages
+2024-08-12 06:30:28,297 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: python_packages
+2024-08-12 06:30:28,297 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-12 06:30:28,298 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 06:30:28,299 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: stop_status
+2024-08-12 06:30:28,566 DEBUG   SenderThread:12721 [sender.py:send():382] send: telemetry
+2024-08-12 06:30:28,941 INFO    wandb-upload_0:12721 [upload_job.py:push():131] Uploaded file /tmp/tmpagb8lhaywandb/h9wwuria-wandb-metadata.json
+2024-08-12 06:30:29,150 INFO    Thread-12 :12721 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_063027-j1htzx7q/files/requirements.txt
+2024-08-12 06:30:29,150 INFO    Thread-12 :12721 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log
+2024-08-12 06:30:29,150 INFO    Thread-12 :12721 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_063027-j1htzx7q/files/wandb-metadata.json
+2024-08-12 06:30:31,151 INFO    Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log
+2024-08-12 06:30:32,987 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:30:37,988 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:30:42,988 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:30:43,296 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-12 06:30:43,297 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 06:30:43,297 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: stop_status
+2024-08-12 06:30:48,530 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:30:53,531 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:30:58,296 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-12 06:30:58,296 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: stop_status
+2024-08-12 06:30:58,340 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 06:30:58,553 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:30:59,169 INFO    Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/config.yaml
+2024-08-12 06:31:03,753 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:31:08,754 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:31:13,296 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-12 06:31:13,297 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: stop_status
+2024-08-12 06:31:13,336 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 06:31:14,505 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:31:19,506 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:31:24,507 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:31:28,255 DEBUG   SystemMonitor:12721 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-08-12 06:31:28,257 DEBUG   SenderThread:12721 [sender.py:send():382] send: stats
+2024-08-12 06:31:28,296 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-12 06:31:28,296 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: stop_status
+2024-08-12 06:31:28,340 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 06:31:30,499 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:31:35,500 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:31:40,561 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:31:41,196 INFO    Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log
+2024-08-12 06:31:43,297 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-12 06:31:43,297 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 06:31:43,298 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: stop_status
+2024-08-12 06:31:46,508 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:31:51,509 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:31:56,510 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:31:58,258 DEBUG   SenderThread:12721 [sender.py:send():382] send: stats
+2024-08-12 06:31:58,296 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-12 06:31:58,297 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: stop_status
+2024-08-12 06:31:58,340 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 06:32:02,497 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:32:07,497 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:32:12,498 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:32:13,297 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-12 06:32:13,297 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: stop_status
+2024-08-12 06:32:13,340 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 06:32:17,531 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:32:22,531 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:32:27,532 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:32:28,259 DEBUG   SenderThread:12721 [sender.py:send():382] send: stats
+2024-08-12 06:32:28,297 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-12 06:32:28,297 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: stop_status
+2024-08-12 06:32:28,340 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 06:32:33,512 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:32:38,513 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:32:43,297 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-12 06:32:43,297 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: stop_status
+2024-08-12 06:32:43,340 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 06:32:43,534 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:32:48,534 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:32:53,535 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:32:58,260 DEBUG   SenderThread:12721 [sender.py:send():382] send: stats
+2024-08-12 06:32:58,297 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-12 06:32:58,297 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: stop_status
+2024-08-12 06:32:58,344 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 06:32:59,520 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:33:01,246 INFO    Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log
+2024-08-12 06:33:05,103 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:33:10,126 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:33:11,252 INFO    Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log
+2024-08-12 06:33:12,515 DEBUG   SenderThread:12721 [sender.py:send():382] send: config
+2024-08-12 06:33:12,515 DEBUG   SenderThread:12721 [sender.py:send():382] send: config
+2024-08-12 06:33:13,253 INFO    Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log
+2024-08-12 06:33:13,297 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-12 06:33:13,298 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: stop_status
+2024-08-12 06:33:13,298 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 06:33:15,255 INFO    Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log
+2024-08-12 06:33:15,581 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:33:15,887 DEBUG   SenderThread:12721 [sender.py:send():382] send: exit
+2024-08-12 06:33:15,887 INFO    SenderThread:12721 [sender.py:send_exit():589] handling exit code: 1
+2024-08-12 06:33:15,887 INFO    SenderThread:12721 [sender.py:send_exit():591] handling runtime: 167
+2024-08-12 06:33:15,889 INFO    SenderThread:12721 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-12 06:33:15,889 INFO    SenderThread:12721 [sender.py:send_exit():597] send defer
+2024-08-12 06:33:15,889 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:15,889 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 0
+2024-08-12 06:33:15,889 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:15,889 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 0
+2024-08-12 06:33:15,889 INFO    SenderThread:12721 [sender.py:transition_state():617] send defer: 1
+2024-08-12 06:33:15,890 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:15,890 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 1
+2024-08-12 06:33:15,890 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:15,890 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 1
+2024-08-12 06:33:15,890 INFO    SenderThread:12721 [sender.py:transition_state():617] send defer: 2
+2024-08-12 06:33:15,890 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:15,890 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 2
+2024-08-12 06:33:15,890 INFO    HandlerThread:12721 [system_monitor.py:finish():203] Stopping system monitor
+2024-08-12 06:33:15,890 INFO    HandlerThread:12721 [interfaces.py:finish():202] Joined cpu monitor
+2024-08-12 06:33:15,890 DEBUG   SystemMonitor:12721 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-08-12 06:33:15,891 INFO    HandlerThread:12721 [interfaces.py:finish():202] Joined disk monitor
+2024-08-12 06:33:15,891 DEBUG   SystemMonitor:12721 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-08-12 06:33:15,924 INFO    HandlerThread:12721 [interfaces.py:finish():202] Joined gpu monitor
+2024-08-12 06:33:15,925 INFO    HandlerThread:12721 [interfaces.py:finish():202] Joined memory monitor
+2024-08-12 06:33:15,925 INFO    HandlerThread:12721 [interfaces.py:finish():202] Joined network monitor
+2024-08-12 06:33:15,925 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:15,925 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 2
+2024-08-12 06:33:15,925 INFO    SenderThread:12721 [sender.py:transition_state():617] send defer: 3
+2024-08-12 06:33:15,926 DEBUG   SenderThread:12721 [sender.py:send():382] send: stats
+2024-08-12 06:33:15,926 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:15,926 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 3
+2024-08-12 06:33:15,926 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:15,926 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 3
+2024-08-12 06:33:15,926 INFO    SenderThread:12721 [sender.py:transition_state():617] send defer: 4
+2024-08-12 06:33:15,926 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:15,926 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 4
+2024-08-12 06:33:15,926 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:15,926 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 4
+2024-08-12 06:33:15,927 INFO    SenderThread:12721 [sender.py:transition_state():617] send defer: 5
+2024-08-12 06:33:15,927 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:15,927 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 5
+2024-08-12 06:33:15,927 DEBUG   SenderThread:12721 [sender.py:send():382] send: summary
+2024-08-12 06:33:15,928 INFO    SenderThread:12721 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-12 06:33:15,928 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:15,928 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 5
+2024-08-12 06:33:15,928 INFO    SenderThread:12721 [sender.py:transition_state():617] send defer: 6
+2024-08-12 06:33:15,928 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:15,928 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 6
+2024-08-12 06:33:15,928 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:15,928 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 6
+2024-08-12 06:33:15,931 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 06:33:16,132 INFO    SenderThread:12721 [sender.py:transition_state():617] send defer: 7
+2024-08-12 06:33:16,132 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:16,132 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 7
+2024-08-12 06:33:16,132 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:16,132 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 7
+2024-08-12 06:33:16,256 INFO    Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/config.yaml
+2024-08-12 06:33:16,256 INFO    Thread-12 :12721 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_063027-j1htzx7q/files/wandb-summary.json
+2024-08-12 06:33:16,887 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-12 06:33:17,100 INFO    SenderThread:12721 [sender.py:transition_state():617] send defer: 8
+2024-08-12 06:33:17,100 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: poll_exit
+2024-08-12 06:33:17,100 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:17,100 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 8
+2024-08-12 06:33:17,100 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:17,100 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 8
+2024-08-12 06:33:17,100 INFO    SenderThread:12721 [job_builder.py:build():296] Attempting to build job artifact
+2024-08-12 06:33:17,101 INFO    SenderThread:12721 [job_builder.py:_get_source_type():426] is repo sourced job
+2024-08-12 06:33:17,116 INFO    SenderThread:12721 [job_builder.py:build():402] adding wandb-job metadata file
+2024-08-12 06:33:17,124 INFO    SenderThread:12721 [sender.py:transition_state():617] send defer: 9
+2024-08-12 06:33:17,125 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:17,125 DEBUG   SenderThread:12721 [sender.py:send():382] send: artifact
+2024-08-12 06:33:17,125 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 9
+2024-08-12 06:33:17,257 INFO    Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log
+2024-08-12 06:33:17,887 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-12 06:33:18,153 INFO    SenderThread:12721 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE0MDAxODM0Nw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE0MDA5NDY1MQ==', 'versionIndex': 9}}}
+2024-08-12 06:33:18,154 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:18,154 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 9
+2024-08-12 06:33:18,154 INFO    SenderThread:12721 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-08-12 06:33:18,258 INFO    SenderThread:12721 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240812_063027-j1htzx7q/files
+2024-08-12 06:33:18,258 INFO    SenderThread:12721 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063027-j1htzx7q/files/requirements.txt requirements.txt
+2024-08-12 06:33:18,258 INFO    SenderThread:12721 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063027-j1htzx7q/files/config.yaml config.yaml
+2024-08-12 06:33:18,259 INFO    SenderThread:12721 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063027-j1htzx7q/files/wandb-metadata.json wandb-metadata.json
+2024-08-12 06:33:18,260 INFO    SenderThread:12721 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063027-j1htzx7q/files/wandb-summary.json wandb-summary.json
+2024-08-12 06:33:18,262 INFO    SenderThread:12721 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log output.log
+2024-08-12 06:33:18,262 INFO    SenderThread:12721 [sender.py:transition_state():617] send defer: 10
+2024-08-12 06:33:18,262 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: poll_exit
+2024-08-12 06:33:18,264 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:18,265 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 10
+2024-08-12 06:33:18,266 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:18,266 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 10
+2024-08-12 06:33:18,266 INFO    SenderThread:12721 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-12 06:33:18,655 INFO    wandb-upload_0:12721 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_063027-j1htzx7q/files/requirements.txt
+2024-08-12 06:33:18,745 INFO    wandb-upload_1:12721 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_063027-j1htzx7q/files/config.yaml
+2024-08-12 06:33:18,843 INFO    wandb-upload_2:12721 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_063027-j1htzx7q/files/wandb-summary.json
+2024-08-12 06:33:18,858 INFO    wandb-upload_3:12721 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_063027-j1htzx7q/files/output.log
+2024-08-12 06:33:18,888 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-12 06:33:18,889 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: poll_exit
+2024-08-12 06:33:19,058 INFO    Thread-11 (_thread_body):12721 [sender.py:transition_state():617] send defer: 11
+2024-08-12 06:33:19,059 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:19,059 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 11
+2024-08-12 06:33:19,059 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:19,059 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 11
+2024-08-12 06:33:19,059 INFO    SenderThread:12721 [file_pusher.py:join():178] waiting for file pusher
+2024-08-12 06:33:19,059 INFO    SenderThread:12721 [sender.py:transition_state():617] send defer: 12
+2024-08-12 06:33:19,059 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:19,059 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 12
+2024-08-12 06:33:19,059 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:19,059 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 12
+2024-08-12 06:33:19,059 INFO    SenderThread:12721 [file_stream.py:finish():595] file stream finish called
+2024-08-12 06:33:19,821 INFO    SenderThread:12721 [file_stream.py:finish():599] file stream finish is done
+2024-08-12 06:33:19,821 INFO    SenderThread:12721 [sender.py:transition_state():617] send defer: 13
+2024-08-12 06:33:19,822 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:19,822 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 13
+2024-08-12 06:33:19,822 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:19,822 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 13
+2024-08-12 06:33:19,822 INFO    SenderThread:12721 [sender.py:transition_state():617] send defer: 14
+2024-08-12 06:33:19,822 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 06:33:19,823 DEBUG   SenderThread:12721 [sender.py:send():382] send: final
+2024-08-12 06:33:19,823 INFO    HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 14
+2024-08-12 06:33:19,823 DEBUG   SenderThread:12721 [sender.py:send():382] send: footer
+2024-08-12 06:33:19,823 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: defer
+2024-08-12 06:33:19,823 INFO    SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 14
+2024-08-12 06:33:19,823 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-12 06:33:19,823 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: poll_exit
+2024-08-12 06:33:19,824 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-12 06:33:19,824 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: poll_exit
+2024-08-12 06:33:19,824 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: server_info
+2024-08-12 06:33:19,824 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: get_summary
+2024-08-12 06:33:19,825 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: server_info
+2024-08-12 06:33:19,826 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: sampled_history
+2024-08-12 06:33:19,826 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 06:33:19,827 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: job_info
+2024-08-12 06:33:19,994 DEBUG   SenderThread:12721 [sender.py:send_request():409] send_request: job_info
+2024-08-12 06:33:19,994 INFO    MainThread:12721 [wandb_run.py:_footer_history_summary_info():3866] rendering history
+2024-08-12 06:33:19,994 INFO    MainThread:12721 [wandb_run.py:_footer_history_summary_info():3898] rendering summary
+2024-08-12 06:33:19,995 INFO    MainThread:12721 [wandb_run.py:_footer_sync_info():3825] logging synced files
+2024-08-12 06:33:19,995 DEBUG   HandlerThread:12721 [handler.py:handle_request():146] handle_request: shutdown
+2024-08-12 06:33:19,995 INFO    HandlerThread:12721 [handler.py:finish():869] shutting down handler
+2024-08-12 06:33:20,827 INFO    WriterThread:12721 [datastore.py:close():296] close: /project/wandb/run-20240812_063027-j1htzx7q/run-j1htzx7q.wandb
+2024-08-12 06:33:20,994 INFO    SenderThread:12721 [sender.py:finish():1572] shutting down sender
+2024-08-12 06:33:20,995 INFO    SenderThread:12721 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-12 06:33:20,995 INFO    SenderThread:12721 [file_pusher.py:join():178] waiting for file pusher

wandb/run-20240812_063027-j1htzx7q/logs/debug.log

@@ -0,0 +1,30 @@
+2024-08-12 06:30:27,595 INFO    MainThread:12650 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
+2024-08-12 06:30:27,595 INFO    MainThread:12650 [wandb_setup.py:_flush():76] Configure stats pid to 12650
+2024-08-12 06:30:27,595 INFO    MainThread:12650 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings
+2024-08-12 06:30:27,595 INFO    MainThread:12650 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings
+2024-08-12 06:30:27,595 INFO    MainThread:12650 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'}
+2024-08-12 06:30:27,595 INFO    MainThread:12650 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-08-12 06:30:27,595 INFO    MainThread:12650 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'}
+2024-08-12 06:30:27,595 INFO    MainThread:12650 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240812_063027-j1htzx7q/logs/debug.log
+2024-08-12 06:30:27,595 INFO    MainThread:12650 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240812_063027-j1htzx7q/logs/debug-internal.log
+2024-08-12 06:30:27,595 INFO    MainThread:12650 [wandb_init.py:init():566] calling init triggers
+2024-08-12 06:30:27,595 INFO    MainThread:12650 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
+config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'valid_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'test_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 4096, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/google/gemma-2-2b', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-sample-gemma-2-2b_train_2024-08-12-06:30:12', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-sample-gemma-2-2b', 'save': '/work/llm_recipes/models/yans-sample-gemma-2-2b', 'base_model': '/share/pretrained_lm/google/gemma-2-2b', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 3, 'save_interval': 3, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 1, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-sample-gemma-2-2b', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 256000, 'gradient_accumulation_steps': 320}
+2024-08-12 06:30:27,596 INFO    MainThread:12650 [wandb_init.py:init():616] starting backend
+2024-08-12 06:30:27,596 INFO    MainThread:12650 [wandb_init.py:init():620] setting up manager
+2024-08-12 06:30:27,600 INFO    MainThread:12650 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-08-12 06:30:27,601 INFO    MainThread:12650 [wandb_init.py:init():628] backend started and connected
+2024-08-12 06:30:27,606 INFO    MainThread:12650 [wandb_init.py:init():720] updated telemetry
+2024-08-12 06:30:27,632 INFO    MainThread:12650 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-08-12 06:30:28,153 INFO    MainThread:12650 [wandb_run.py:_on_init():2262] communicating current version
+2024-08-12 06:30:28,237 INFO    MainThread:12650 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.6 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-08-12 06:30:28,237 INFO    MainThread:12650 [wandb_init.py:init():804] starting run threads in backend
+2024-08-12 06:30:28,296 INFO    MainThread:12650 [wandb_run.py:_console_start():2241] atexit reg
+2024-08-12 06:30:28,296 INFO    MainThread:12650 [wandb_run.py:_redirect():2096] redirect: wrap_raw
+2024-08-12 06:30:28,296 INFO    MainThread:12650 [wandb_run.py:_redirect():2161] Wrapping output streams.
+2024-08-12 06:30:28,296 INFO    MainThread:12650 [wandb_run.py:_redirect():2186] Redirects installed.
+2024-08-12 06:30:28,298 INFO    MainThread:12650 [wandb_init.py:init():847] run started, returning control to user process
+2024-08-12 06:33:12,514 INFO    MainThread:12650 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Gemma2ForCausalLM', 'activation_function': 'gelu_pytorch_tanh', 'hidden_size': 2304, 'model_type': 'gemma2', 'max_position_embeddings': 4096, 'num_attention_heads': 8, 'num_hidden_layers': 26}
+2024-08-12 06:33:12,515 INFO    MainThread:12650 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1}
+2024-08-12 06:33:20,996 WARNING MsgRouterThr:12650 [router.py:message_loop():77] message_loop has been closed

wandb/run-20240823_163849-faey1t8u/files/config.yaml

@@ -0,0 +1,342 @@
+wandb_version: 1
+
+sharding_strategy:
+  desc: null
+  value: FULL_SHARD
+checkpoint_type:
+  desc: null
+  value: LOCAL_STATE_DICT
+fsdp_activation_checkpointing:
+  desc: null
+  value: true
+fsdp_cpu_offload:
+  desc: null
+  value: false
+low_cpu_fsdp:
+  desc: null
+  value: false
+no_meta_device:
+  desc: null
+  value: false
+data_path:
+  desc: null
+  value: null
+split:
+  desc: null
+  value: 969, 30, 1
+train_data_path:
+  desc: null
+  value:
+  - '1754785366'
+  - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document
+  - '28623823675'
+  - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document
+valid_data_path:
+  desc: null
+  value:
+  - '1754785366'
+  - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document
+test_data_path:
+  desc: null
+  value:
+  - '1754785366'
+  - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document
+data_cache_path:
+  desc: null
+  value: null
+vocab_size:
+  desc: null
+  value: null
+vocab_file:
+  desc: null
+  value: null
+merge_file:
+  desc: null
+  value: null
+seq_length:
+  desc: null
+  value: 4096
+num_workers:
+  desc: null
+  value: 2
+tokenizer_type:
+  desc: null
+  value: HFPreTrainedTokenizer
+tokenizer_model:
+  desc: null
+  value: /share/pretrained_lm/Qwen/Qwen2-0.5B
+reset_position_ids:
+  desc: null
+  value: false
+reset_attention_mask:
+  desc: null
+  value: false
+eod_mask_loss:
+  desc: null
+  value: false
+retro_return_doc_ids:
+  desc: null
+  value: false
+short_seq_prob:
+  desc: null
+  value: 0.1
+vocab_extra_ids:
+  desc: null
+  value: 0
+seed:
+  desc: null
+  value: 1234
+use_mpi:
+  desc: null
+  value: false
+wandb_entity:
+  desc: null
+  value: iwakawa-koichi-q5-tohoku-nlp6723
+wandb_name:
+  desc: null
+  value: Qwen2-0.5b-0.2_train_2024-08-23-16:38:35
+wandb_project:
+  desc: null
+  value: llm_tutorial-0.2
+quantization:
+  desc: null
+  value: false
+use_freeze_layers:
+  desc: null
+  value: false
+freeze_layers:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+mixed_precision:
+  desc: null
+  value: true
+param_dtype:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: /work/llm_recipes/models/Qwen2-0.5b-0.2
+save:
+  desc: null
+  value: /work/llm_recipes/models/Qwen2-0.5b-0.2
+base_model:
+  desc: null
+  value: /share/pretrained_lm/Qwen/Qwen2-0.5B
+use_better_transformer:
+  desc: null
+  value: false
+grad_clip_norm:
+  desc: null
+  value: 1.0
+eval_interval:
+  desc: null
+  value: 10
+save_interval:
+  desc: null
+  value: 10
+eval_iters:
+  desc: null
+  value: 10
+optimizer:
+  desc: null
+  value: anyprecision
+lr:
+  desc: null
+  value: 2.0e-05
+lr_decay_style:
+  desc: null
+  value: cosine
+lr_decay_iters:
+  desc: null
+  value: 7500
+lr_warmup_iters:
+  desc: null
+  value: 500
+min_lr:
+  desc: null
+  value: 1.0e-06
+train_iters:
+  desc: null
+  value: 7500
+train_samples:
+  desc: null
+  value: null
+global_batch_size:
+  desc: null
+  value: 320
+micro_batch_size:
+  desc: null
+  value: 3
+make_vocab_size_divisible_by:
+  desc: null
+  value: 128
+sliding_window_size:
+  desc: null
+  value: 131072
+skip_batch:
+  desc: null
+  value: null
+no_save_optimizer_state:
+  desc: null
+  value: false
+continual_pretraining:
+  desc: null
+  value: false
+instruction_tuning:
+  desc: null
+  value: false
+direct_preference_optimization:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.1
+hidden_dropout:
+  desc: null
+  value: 0.1
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_eps:
+  desc: null
+  value: 1.0e-06
+hf_transformer_model_dir:
+  desc: null
+  value: null
+instruction_train_data_path:
+  desc: null
+  value: null
+instruction_valid_data_path:
+  desc: null
+  value: null
+epoch:
+  desc: null
+  value: null
+instruction_dataset_size:
+  desc: null
+  value: null
+save_sampler_state:
+  desc: null
+  value: false
+label_smoothing:
+  desc: null
+  value: 0.0
+save_n_checkpoints:
+  desc: null
+  value: 10
+hf_repo_id:
+  desc: null
+  value: koichi12/Qwen2-0.5b-0.2
+create_public_hf_repo:
+  desc: null
+  value: false
+upload_all_checkpoints_to_hf:
+  desc: null
+  value: true
+hf_upload_retry_limit:
+  desc: null
+  value: 2
+exit_duration_in_mins:
+  desc: null
+  value: null
+source_key:
+  desc: null
+  value: null
+target_key:
+  desc: null
+  value: null
+attn_implementation:
+  desc: null
+  value: flash_attention_2
+efficient_instruction_tuning:
+  desc: null
+  value: false
+remove_padding_masking:
+  desc: null
+  value: false
+save_start_iter:
+  desc: null
+  value: null
+valid_micro_batch_size:
+  desc: null
+  value: 1
+rank:
+  desc: null
+  value: 0
+world_size:
+  desc: null
+  value: 1
+padded_vocab_size:
+  desc: null
+  value: 151680
+gradient_accumulation_steps:
+  desc: null
+  value: 106
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.43.3
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1724398729.364923
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      - 105
+      2:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      - 105
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.43.3
+      8:
+      - 5
+      13: linux-x86_64
+model_architecture:
+  desc: null
+  value: Qwen2ForCausalLM
+activation_function:
+  desc: null
+  value: silu
+hidden_size:
+  desc: null
+  value: 896
+model_type:
+  desc: null
+  value: qwen2
+max_position_embeddings:
+  desc: null
+  value: 4096
+num_attention_heads:
+  desc: null
+  value: 14
+num_hidden_layers:
+  desc: null
+  value: 24

wandb/run-20240823_163849-faey1t8u/files/output.log

@@ -0,0 +1,126 @@
+Created Hugging Face repository with ID koichi12/Qwen2-0.5b-0.2.
+Clearing GPU cache for all ranks
+--> Running with torch torch_distributed debug set to detail
+Loading model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+Loaded model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt
+--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B
+--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params
+BFloat16 enabled for mixed precision - using bfSixteen policy
+--> applying fsdp activation checkpointing...
+ > datasets target sizes (minimum size):
+    train:      2400000
+    validation: 2403200
+    test:       3200
+> building train, validation, and test datasets for GPT ...
+/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1.
+  warnings.warn(
+Let split = None
+Unable to save the indexes because path_to_cache is None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+> finished creating GPT datasets ...
+Loading optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt
+[rank0]:[2024-08-23 16:38:58,062] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling:  defaultdict(<class 'float'>, {})
+Loaded optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt
+model info: FullyShardedDataParallel(
+  (_fsdp_wrapped_module): Qwen2ForCausalLM(
+    (model): Qwen2Model(
+      (embed_tokens): Embedding(151936, 896)
+      (layers): ModuleList(
+        (0-23): 24 x FullyShardedDataParallel(
+          (_fsdp_wrapped_module): CheckpointWrapper(
+            (_checkpoint_wrapped_module): Qwen2DecoderLayer(
+              (self_attn): Qwen2FlashAttention2(
+                (q_proj): Linear(in_features=896, out_features=896, bias=True)
+                (k_proj): Linear(in_features=896, out_features=128, bias=True)
+                (v_proj): Linear(in_features=896, out_features=128, bias=True)
+                (o_proj): Linear(in_features=896, out_features=896, bias=False)
+                (rotary_emb): Qwen2RotaryEmbedding()
+              )
+              (mlp): Qwen2MLP(
+                (gate_proj): Linear(in_features=896, out_features=4864, bias=False)
+                (up_proj): Linear(in_features=896, out_features=4864, bias=False)
+                (down_proj): Linear(in_features=4864, out_features=896, bias=False)
+                (act_fn): SiLU()
+              )
+              (input_layernorm): Qwen2RMSNorm()
+              (post_attention_layernorm): Qwen2RMSNorm()
+            )
+          )
+        )
+      )
+      (norm): Qwen2RMSNorm()
+    )
+    (lm_head): Linear(in_features=896, out_features=151936, bias=False)
+  )
+)
+model config: Qwen2Config {
+  "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "label_smoothing": 0.0,
+  "max_position_embeddings": 4096,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.3",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}
+------------------------------------------------------------------
+iteration: 41 , TFLOPS: 89.20992749379542, Tokens per sec: 22186.06078570857, Loss: 4.376823425292969
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 42 , TFLOPS: 90.73548605717187, Tokens per sec: 22565.459536162463, Loss: 4.388589382171631
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 43 , TFLOPS: 90.773373714071, Tokens per sec: 22574.882006089374, Loss: 4.334207057952881
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 44 , TFLOPS: 90.78534040748795, Tokens per sec: 22577.85806262267, Loss: 4.347831726074219
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 45 , TFLOPS: 90.91842135677658, Tokens per sec: 22610.954626125003, Loss: 4.369765281677246
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 46 , TFLOPS: 90.73681564901436, Tokens per sec: 22565.79019897395, Loss: 4.371013164520264
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 47 , TFLOPS: 90.85118295674725, Tokens per sec: 22594.23277383689, Loss: 4.347028732299805
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 48 , TFLOPS: 90.81419242163798, Tokens per sec: 22585.033413592068, Loss: 4.319859504699707
+------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/project/examples/finetuning.py", line 13, in <module>
+    main()
+  File "/project/src/llama_recipes/finetuning.py", line 282, in main
+    train(
+  File "/project/src/llama_recipes/utils/train_utils.py", line 118, in train
+    loss.backward()
+  File "/usr/local/lib/python3.10/dist-packages/torch/_tensor.py", line 522, in backward
+    torch.autograd.backward(
+  File "/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py", line 267, in backward
+    _engine_run_backward(
+  File "/usr/local/lib/python3.10/dist-packages/torch/autograd/graph.py", line 681, in _engine_run_backward
+    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+KeyboardInterrupt

wandb/run-20240823_163849-faey1t8u/files/requirements.txt

@@ -0,0 +1,375 @@
+absl-py==2.1.0
+accelerate==0.23.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+apex==0.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+astroid==3.2.4
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+beautifulsoup4==4.12.3
+bert-score==0.3.13
+bleach==6.1.0
+blis==0.7.11
+build==1.2.1
+cachecontrol==0.14.0
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+cleo==2.1.0
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+cmake==3.28.1
+colorama==0.4.6
+comm==0.2.1
+confection==0.1.4
+contourpy==1.2.0
+cramjam==2.8.3
+crashtest==0.4.1
+cryptography==43.0.0
+cubinlinker==0.3.0+2.g405ac64
+cuda-python==12.3.0rc4+9.gdb8c48a.dirty
+cudf==23.12.0
+cugraph-dgl==23.12.0
+cugraph-service-client==23.12.0
+cugraph-service-server==23.12.0
+cugraph==23.12.0
+cuml==23.12.0
+cupy-cuda12x==12.3.0
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.8
+dask-cuda==23.12.0
+dask-cudf==23.12.0
+dask==2023.11.0
+dataclasses-json==0.6.7
+dataproperty==1.0.1
+datasets==2.20.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.8
+distlib==0.3.8
+distributed==2023.11.0
+distro==1.9.0
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+dulwich==0.21.7
+einops==0.7.0
+emoji==2.12.1
+entmax==1.3
+evaluate==0.4.2
+exceptiongroup==1.2.0
+execnet==2.0.2
+executing==2.0.1
+expecttest==0.1.3
+fastjsonschema==2.19.1
+fastparquet==2023.10.1
+fastrlock==0.8.2
+filelock==3.13.1
+flash-attn==2.4.2
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2023.12.2
+fugashi==1.3.2
+fuzzywuzzy==0.18.0
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.27.0
+graphsurgeon==0.4.6
+greenlet==3.0.3
+grpcio==1.60.1
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.24.5
+hydra-core==1.3.2
+hypothesis==5.35.1
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+installer==0.7.0
+intel-openmp==2021.4.0
+ipadic==1.0.0
+ipykernel==6.29.2
+ipython-genutils==0.2.0
+ipython==8.21.0
+isort==5.13.2
+jaraco.classes==3.4.0
+jedi==0.19.1
+jeepney==0.8.0
+jinja2==3.1.3
+jiter==0.5.0
+joblib==1.3.2
+json5==0.9.14
+jsonargparse==3.13.1
+jsonlines==4.0.0
+jsonnet==0.19.1
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema-specifications==2023.12.1
+jsonschema==4.21.1
+jupyter-client==8.6.0
+jupyter-core==5.7.1
+jupyter-tensorboard==0.2.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==1.2.0
+jupyterlab==2.3.2
+jupytext==1.16.1
+keyring==24.3.1
+kiwisolver==1.4.5
+langchain-community==0.2.12
+langchain-core==0.2.31
+langchain-huggingface==0.0.2
+langchain-openai==0.1.21
+langchain-text-splitters==0.2.2
+langchain==0.2.13
+langcodes==3.3.0
+langsmith==0.1.99
+lazy-loader==0.3
+levenshtein==0.25.1
+librosa==0.10.1
+lightning-utilities==0.11.6
+llm-jp-eval==1.4.0
+llvmlite==0.40.1
+lm-eval==0.3.0
+locket==1.0.0
+logzero==1.7.0
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown==3.5.2
+markupsafe==2.1.4
+marshmallow==3.21.3
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mbstrdecoder==1.1.3
+mccabe==0.7.0
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mecab-python3==1.0.6
+mistune==3.0.2
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mkl==2021.1.1
+mock==5.1.0
+mojimoji==0.0.13
+more-itertools==9.1.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+multiprocess==0.70.16
+murmurhash==1.0.10
+mypy-extensions==1.0.0
+nbclient==0.9.0
+nbconvert==7.16.0
+nbformat==5.9.2
+neologdn==0.5.3
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.4.10
+numba==0.57.1+1.g1ff679645
+numexpr==2.10.1
+numpy==1.24.4
+nvfuser==0.1.4a0+d0bb811
+nvidia-dali-cuda120==1.34.0
+nvidia-pyindex==1.0.9
+nvtx==0.2.5
+oauthlib==3.2.2
+omegaconf==2.3.0
+onnx==1.15.0rc2
+openai==1.40.6
+opencv==4.7.0
+optree==0.10.0
+orjson==3.10.7
+packaging==23.2
+pandas==2.2.2
+pandocfilters==1.5.1
+parso==0.8.3
+partd==1.4.1
+pathvalidate==3.2.0
+peft==0.5.0
+pexpect==4.9.0
+pillow==10.2.0
+pip==24.0
+pkginfo==1.11.1
+plac==1.4.3
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+poetry-core==1.9.0
+poetry-plugin-export==1.8.0
+poetry==1.8.3
+polygraphy==0.49.4
+pooch==1.8.0
+portalocker==2.10.1
+preshed==3.0.9
+prettytable==3.9.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.24.4
+psutil==5.9.4
+ptxcompiler==0.8.1+2.g0d406d6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow-hotfix==0.6
+pyarrow==15.0.2
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pybind11-global==2.11.1
+pybind11==2.11.1
+pycocotools==2.0+nv0.8.0
+pycountry==24.6.1
+pycparser==2.21
+pydantic-core==2.16.2
+pydantic==2.6.1
+pygments==2.17.2
+pylibcugraph==23.12.0
+pylibcugraphops==23.12.0
+pylibraft==23.12.0
+pylint==3.2.6
+pynvml==11.4.1
+pyparsing==3.1.1
+pyproject-hooks==1.1.0
+pytablewriter==1.2.0
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==13.0
+pytest-shard==0.1.2
+pytest-xdist==3.5.0
+pytest==8.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-hostlist==1.23.0
+python-levenshtein==0.25.1
+pytorch-lightning==2.4.0
+pytorch-quantization==2.1.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.2
+raft-dask==23.12.0
+rapidfuzz==3.9.6
+rapids-dask-dependency==23.12.1
+referencing==0.33.0
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests-toolbelt==1.0.0
+requests==2.32.3
+rhoknp==1.7.0
+rich==13.7.0
+rmm==23.12.0
+rouge-score==0.1.2
+rpds-py==0.17.1
+rsa==4.9
+sacrebleu==2.4.2
+safetensors==0.4.3
+scikit-learn==1.5.1
+scipy==1.12.0
+secretstorage==3.3.3
+send2trash==1.8.2
+sentence-transformers==3.0.1
+sentencepiece==0.1.99
+sentry-sdk==2.12.0
+setproctitle==1.3.3
+setuptools==68.2.2
+shellingham==1.5.4
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sniffio==1.3.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sphinx-glpi-theme==0.6
+sqlalchemy==2.0.32
+sqlitedict==2.1.0
+srsly==2.4.8
+stack-data==0.6.3
+sumeval==0.2.2
+sympy==1.12
+tabledata==1.3.3
+tabulate==0.9.0
+tbb==2021.11.0
+tblib==3.0.0
+tcolorpy==0.1.6
+tenacity==8.5.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.9.0
+tensorrt==8.6.3
+terminado==0.18.0
+termplotlib==0.3.9
+text-generation==0.7.0
+thinc==8.2.3
+threadpoolctl==3.2.0
+thriftpy2==0.4.17
+tiktoken==0.7.0
+tinycss2==1.2.1
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+tomlkit==0.13.2
+toolz==0.12.1
+torch-tensorrt==2.3.0a0
+torch==2.3.0a0+ebedce2
+torchdata==0.7.1a0
+torchmetrics==0.10.3
+torchtext==0.17.0a0
+torchvision==0.18.0a0
+tornado==6.4
+tqdm-multiprocess==0.0.11
+tqdm==4.66.5
+traitlets==5.9.0
+transformer-engine==1.3.0+5b90b7f
+transformers==4.43.3
+treelite-runtime==3.9.1
+treelite==3.9.1
+triton==2.2.0+e28a256
+trove-classifiers==2024.7.2
+typepy==1.3.2
+typer==0.9.0
+types-dataclasses==0.6.6
+typing-extensions==4.12.2
+typing-inspect==0.9.0
+tzdata==2024.1
+ucx-py==0.35.0
+uff==0.6.9
+ujson==5.8.0
+unbabel-comet==2.2.2
+unidic-lite==1.0.8
+urllib3==1.26.18
+virtualenv==20.26.3
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.3.4
+webencodings==0.5.1
+werkzeug==3.0.1
+wheel==0.42.0
+word2number==1.1
+xdoctest==1.0.2
+xgboost==1.7.6
+xmltodict==0.13.0
+xxhash==3.4.1
+yarl==1.9.4
+zict==3.0.0
+zipp==3.17.0
+zstandard==0.23.0

wandb/run-20240823_163849-faey1t8u/files/wandb-metadata.json

@@ -0,0 +1,220 @@
+{
+    "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-08-23T07:38:49.894670",
+    "startedAt": "2024-08-23T07:38:49.352718",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--seq-length",
+        "4096",
+        "--sliding-window-size",
+        "131072",
+        "--micro-batch-size",
+        "3",
+        "--valid_micro_batch_size",
+        "1",
+        "--global-batch-size",
+        "320",
+        "--train-iters",
+        "7500",
+        "--tokenizer-type",
+        "HFPreTrainedTokenizer",
+        "--tokenizer-model",
+        "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+        "--train-data-path",
+        "1754785366",
+        "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document",
+        "28623823675",
+        "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document",
+        "--valid-data-path",
+        "1754785366",
+        "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document",
+        "--test-data-path",
+        "1754785366",
+        "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document",
+        "--lr",
+        "2e-5",
+        "--min-lr",
+        "1e-6",
+        "--lr-decay-style",
+        "cosine",
+        "--lr-warmup-iters",
+        "500",
+        "--lr-decay-iters",
+        "7500",
+        "--weight-decay",
+        "0.1",
+        "--grad-clip-norm",
+        "1.0",
+        "--optimizer",
+        "anyprecision",
+        "--adam-beta1",
+        "0.9",
+        "--adam-beta2",
+        "0.95",
+        "--adam-eps",
+        "1e-6",
+        "--save-interval",
+        "10",
+        "--eval-interval",
+        "10",
+        "--eval-iters",
+        "10",
+        "--bf16",
+        "--mixed-precision",
+        "--base-model",
+        "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+        "--save",
+        "/work/llm_recipes/models/Qwen2-0.5b-0.2",
+        "--load",
+        "/work/llm_recipes/models/Qwen2-0.5b-0.2",
+        "--fsdp-activation-checkpointing",
+        "--sharding-strategy",
+        "FULL_SHARD",
+        "--checkpoint-type",
+        "LOCAL_STATE_DICT",
+        "--save-n-checkpoints",
+        "10",
+        "--upload-all-checkpoints-to-hf",
+        "--hf-upload-retry-limit",
+        "2",
+        "--hf-repo-id",
+        "koichi12/Qwen2-0.5b-0.2",
+        "--wandb-entity",
+        "iwakawa-koichi-q5-tohoku-nlp6723",
+        "--wandb-project",
+        "llm_tutorial-0.2",
+        "--wandb-name",
+        "Qwen2-0.5b-0.2_train_2024-08-23-16:38:35"
+    ],
+    "state": "running",
+    "program": "/project/examples/finetuning.py",
+    "codePathLocal": "examples/finetuning.py",
+    "codePath": "examples/finetuning.py",
+    "git": {
+        "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git",
+        "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6"
+    },
+    "email": null,
+    "root": "/project",
+    "host": "gpu-koiwa-00",
+    "username": "koiwa",
+    "executable": "/usr/bin/python",
+    "cpu_count": 18,
+    "cpu_count_logical": 18,
+    "cpu_freq": {
+        "current": 2400.0389999999993,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 0.0625,
+            "used": 1.1444091796875e-05
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 56.487831115722656
+    }
+}

wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json

@@ -0,0 +1 @@
+{"training/loss": 4.319859504699707, "training/perplexity": 75.17806538514934, "utils/batch_size": 3, "utils/global_batch_size": 318, "utils/seq_len": 4097, "utils/gradient_accumulation_steps": 106, "utils/iteration": 48, "optimizer/lr": 2.8240000000000004e-06, "optimizer/variance_l2": 0.050272112510912605, "optimizer/variance_sqrt_l2": 0.9579955556165142, "optimizer/momentum_l2": 0.9571913293356521, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 0.9176788330078125, "optimizer/variance_sqrt_l1": 4099.5, "optimizer/momentum_l1": 3579.0, "optimizer/weight_l1": 6886400.0, "optimizer/variance_abs_max": 0.0419921875, "optimizer/variance_sqrt_abs_max": 0.205078125, "optimizer/momentum_abs_max": 0.2236328125, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 57.68625514700034, "stats/tokens_per_sec": 22585.033413592068, "stats/tokens_per_sec_per_gpu": 22585.033413592068, "stats/tflops": 90.81419242163798, "_timestamp": 1724399202.2699971, "_runtime": 472.90507411956787, "_step": 48, "_wandb": {"runtime": 476}}

wandb/run-20240823_163849-faey1t8u/logs/debug-internal.log

@@ -0,0 +1,439 @@
+2024-08-23 16:38:49,367 INFO    StreamThr :12305 [internal.py:wandb_internal():86] W&B internal server running at pid: 12305, started at: 2024-08-23 16:38:49.366185
+2024-08-23 16:38:49,368 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status
+2024-08-23 16:38:49,370 INFO    WriterThread:12305 [datastore.py:open_for_write():87] open: /project/wandb/run-20240823_163849-faey1t8u/run-faey1t8u.wandb
+2024-08-23 16:38:49,371 DEBUG   SenderThread:12305 [sender.py:send():382] send: header
+2024-08-23 16:38:49,405 DEBUG   SenderThread:12305 [sender.py:send():382] send: run
+2024-08-23 16:38:49,800 INFO    SenderThread:12305 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240823_163849-faey1t8u/files
+2024-08-23 16:38:49,800 INFO    SenderThread:12305 [sender.py:_start_run_threads():1136] run started: faey1t8u with start time 1724398729.364923
+2024-08-23 16:38:49,806 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: check_version
+2024-08-23 16:38:49,806 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: check_version
+2024-08-23 16:38:49,876 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: run_start
+2024-08-23 16:38:49,882 DEBUG   HandlerThread:12305 [system_info.py:__init__():27] System info init
+2024-08-23 16:38:49,882 DEBUG   HandlerThread:12305 [system_info.py:__init__():42] System info init done
+2024-08-23 16:38:49,882 INFO    HandlerThread:12305 [system_monitor.py:start():194] Starting system monitor
+2024-08-23 16:38:49,882 INFO    SystemMonitor:12305 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-08-23 16:38:49,882 INFO    HandlerThread:12305 [system_monitor.py:probe():214] Collecting system info
+2024-08-23 16:38:49,883 INFO    SystemMonitor:12305 [interfaces.py:start():190] Started cpu monitoring
+2024-08-23 16:38:49,883 INFO    SystemMonitor:12305 [interfaces.py:start():190] Started disk monitoring
+2024-08-23 16:38:49,884 INFO    SystemMonitor:12305 [interfaces.py:start():190] Started gpu monitoring
+2024-08-23 16:38:49,885 INFO    SystemMonitor:12305 [interfaces.py:start():190] Started memory monitoring
+2024-08-23 16:38:49,886 INFO    SystemMonitor:12305 [interfaces.py:start():190] Started network monitoring
+2024-08-23 16:38:49,894 DEBUG   HandlerThread:12305 [system_info.py:probe():151] Probing system
+2024-08-23 16:38:49,896 DEBUG   HandlerThread:12305 [system_info.py:_probe_git():136] Probing git
+2024-08-23 16:38:49,908 DEBUG   HandlerThread:12305 [system_info.py:_probe_git():144] Probing git done
+2024-08-23 16:38:49,909 DEBUG   HandlerThread:12305 [system_info.py:probe():199] Probing system done
+2024-08-23 16:38:49,909 DEBUG   HandlerThread:12305 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-23T07:38:49.894670', 'startedAt': '2024-08-23T07:38:49.352718', 'docker': None, 'cuda': None, 'args': ('--seq-length', '4096', '--sliding-window-size', '131072', '--micro-batch-size', '3', '--valid_micro_batch_size', '1', '--global-batch-size', '320', '--train-iters', '7500', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document', '--valid-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--test-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '7500', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '10', '--eval-interval', '10', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--load', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/Qwen2-0.5b-0.2', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial-0.2', '--wandb-name', 'Qwen2-0.5b-0.2_train_2024-08-23-16:38:35'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '887a2cc5d104c10264701f95cbbb0a6a116768d6'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0389999999993, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.487831115722656}}
+2024-08-23 16:38:49,909 INFO    HandlerThread:12305 [system_monitor.py:probe():224] Finished collecting system info
+2024-08-23 16:38:49,909 INFO    HandlerThread:12305 [system_monitor.py:probe():227] Publishing system info
+2024-08-23 16:38:49,910 INFO    HandlerThread:12305 [system_monitor.py:probe():229] Finished publishing system info
+2024-08-23 16:38:49,915 DEBUG   SenderThread:12305 [sender.py:send():382] send: files
+2024-08-23 16:38:49,916 INFO    SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now
+2024-08-23 16:38:49,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: python_packages
+2024-08-23 16:38:49,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:38:49,928 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:38:49,928 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: python_packages
+2024-08-23 16:38:49,930 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:38:50,183 DEBUG   SenderThread:12305 [sender.py:send():382] send: telemetry
+2024-08-23 16:38:50,520 INFO    wandb-upload_0:12305 [upload_job.py:push():131] Uploaded file /tmp/tmpljn_2vd6wandb/tfd8n6zw-wandb-metadata.json
+2024-08-23 16:38:50,802 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-metadata.json
+2024-08-23 16:38:50,802 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_163849-faey1t8u/files/requirements.txt
+2024-08-23 16:38:50,803 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:38:52,803 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:38:54,804 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:38:54,958 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:38:56,805 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:38:58,352 DEBUG   SenderThread:12305 [sender.py:send():382] send: config
+2024-08-23 16:38:58,353 DEBUG   SenderThread:12305 [sender.py:send():382] send: config
+2024-08-23 16:38:58,807 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:39:00,353 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:39:00,808 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:39:04,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:39:04,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:39:04,928 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:39:06,113 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:39:11,113 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:39:16,114 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:39:19,926 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:39:19,926 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:39:19,967 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:39:21,162 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:39:21,819 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/config.yaml
+2024-08-23 16:39:26,345 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:39:31,346 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:39:34,926 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:39:34,926 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:39:34,967 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:39:37,125 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:39:42,126 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:39:47,127 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:39:49,886 DEBUG   SystemMonitor:12305 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-08-23 16:39:49,888 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:39:49,926 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:39:49,926 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:39:49,967 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:39:53,111 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:39:58,112 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:39:58,398 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:40:00,838 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:40:03,440 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:40:04,926 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:40:04,927 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:40:04,928 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:40:09,115 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:40:14,116 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:40:19,117 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:40:19,889 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:40:19,926 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:40:19,927 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:40:19,967 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:40:25,115 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:40:30,116 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:40:34,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:40:34,927 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:40:34,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:40:35,173 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:40:40,174 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:40:45,174 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:40:49,890 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:40:49,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:40:49,927 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:40:49,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:40:51,147 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:40:56,136 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:40:56,139 DEBUG   SenderThread:12305 [sender.py:send():382] send: history
+2024-08-23 16:40:56,139 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:40:56,154 INFO    SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:40:56,154 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:40:56,867 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json
+2024-08-23 16:40:58,868 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:41:01,180 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:41:04,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:41:04,927 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:41:04,929 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:41:07,173 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:41:12,174 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:41:17,174 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:41:19,891 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:41:19,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:41:19,927 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:41:19,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:41:23,138 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:41:28,138 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:41:33,139 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:41:34,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:41:34,927 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:41:34,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:41:39,118 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:41:44,119 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:41:49,120 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:41:49,892 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:41:49,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:41:49,927 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:41:49,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:41:53,851 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:41:53,852 DEBUG   SenderThread:12305 [sender.py:send():382] send: history
+2024-08-23 16:41:53,852 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:41:53,854 INFO    SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:41:53,896 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json
+2024-08-23 16:41:54,892 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:41:54,897 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:41:59,892 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:42:04,893 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:42:04,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:42:04,927 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:42:04,929 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:42:10,105 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:42:15,106 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:42:19,893 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:42:19,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:42:19,927 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:42:19,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:42:20,146 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:42:25,146 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:42:30,147 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:42:34,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:42:34,928 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:42:34,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:42:36,138 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:42:41,138 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:42:46,139 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:42:49,894 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:42:49,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:42:49,928 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:42:49,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:42:51,557 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:42:51,559 DEBUG   SenderThread:12305 [sender.py:send():382] send: history
+2024-08-23 16:42:51,559 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:42:51,560 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:42:51,561 INFO    SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:42:51,926 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json
+2024-08-23 16:42:52,927 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:42:56,600 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:43:01,601 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:43:04,927 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:43:04,928 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:43:04,930 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:43:07,132 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:43:12,132 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:43:17,132 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:43:19,895 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:43:19,928 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:43:19,928 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:43:19,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:43:23,107 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:43:28,108 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:43:33,108 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:43:34,928 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:43:34,928 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:43:34,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:43:38,202 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:43:43,203 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:43:48,203 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:43:49,180 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:43:49,181 DEBUG   SenderThread:12305 [sender.py:send():382] send: history
+2024-08-23 16:43:49,182 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:43:49,184 INFO    SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:43:49,896 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:43:49,928 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:43:49,928 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:43:49,930 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:43:49,954 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json
+2024-08-23 16:43:50,955 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:43:54,116 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:43:59,116 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:44:04,117 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:44:04,928 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:44:04,928 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:44:04,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:44:09,123 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:44:14,124 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:44:19,124 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:44:19,897 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:44:19,928 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:44:19,928 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:44:19,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:44:25,120 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:44:30,120 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:44:34,928 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:44:34,929 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:44:34,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:44:35,144 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:44:40,145 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:44:45,145 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:44:46,917 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:44:46,920 DEBUG   SenderThread:12305 [sender.py:send():382] send: history
+2024-08-23 16:44:46,921 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:44:46,922 INFO    SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:44:46,983 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json
+2024-08-23 16:44:48,984 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:44:49,898 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:44:49,928 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:44:49,928 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:44:49,931 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:44:50,159 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:44:55,160 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:45:00,160 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:45:04,928 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:45:04,929 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:45:04,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:45:06,142 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:45:11,142 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:45:16,142 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:45:19,899 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:45:19,928 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:45:19,929 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:45:19,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:45:21,199 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:45:26,199 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:45:31,200 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:45:34,928 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:45:34,929 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:45:34,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:45:37,098 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:45:42,098 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:45:44,582 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:45:44,584 DEBUG   SenderThread:12305 [sender.py:send():382] send: history
+2024-08-23 16:45:44,584 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:45:44,585 INFO    SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:45:45,011 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json
+2024-08-23 16:45:47,011 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:45:47,624 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:45:49,899 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:45:49,928 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:45:49,929 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:45:49,931 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:45:53,153 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:45:58,153 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:46:03,154 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:46:04,929 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:46:04,929 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:46:04,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:46:09,099 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:46:14,100 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:46:19,101 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:46:19,900 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:46:19,929 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:46:19,929 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:46:19,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:46:24,145 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:46:29,146 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:46:34,146 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:46:34,929 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:46:34,929 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:46:34,971 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:46:40,145 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:46:42,271 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:46:42,273 DEBUG   SenderThread:12305 [sender.py:send():382] send: history
+2024-08-23 16:46:42,273 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:46:42,274 INFO    SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:46:43,038 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json
+2024-08-23 16:46:45,038 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:46:45,316 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:46:46,081 DEBUG   SenderThread:12305 [sender.py:send():382] send: exit
+2024-08-23 16:46:46,081 INFO    SenderThread:12305 [sender.py:send_exit():589] handling exit code: 255
+2024-08-23 16:46:46,081 INFO    SenderThread:12305 [sender.py:send_exit():591] handling runtime: 476
+2024-08-23 16:46:46,083 INFO    SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:46:46,083 INFO    SenderThread:12305 [sender.py:send_exit():597] send defer
+2024-08-23 16:46:46,083 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:46,083 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 0
+2024-08-23 16:46:46,083 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:46,083 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 0
+2024-08-23 16:46:46,083 INFO    SenderThread:12305 [sender.py:transition_state():617] send defer: 1
+2024-08-23 16:46:46,083 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:46,083 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 1
+2024-08-23 16:46:46,084 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:46,084 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 1
+2024-08-23 16:46:46,084 INFO    SenderThread:12305 [sender.py:transition_state():617] send defer: 2
+2024-08-23 16:46:46,084 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:46,084 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 2
+2024-08-23 16:46:46,084 INFO    HandlerThread:12305 [system_monitor.py:finish():203] Stopping system monitor
+2024-08-23 16:46:46,084 DEBUG   SystemMonitor:12305 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-08-23 16:46:46,084 INFO    HandlerThread:12305 [interfaces.py:finish():202] Joined cpu monitor
+2024-08-23 16:46:46,084 DEBUG   SystemMonitor:12305 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-08-23 16:46:46,084 INFO    HandlerThread:12305 [interfaces.py:finish():202] Joined disk monitor
+2024-08-23 16:46:46,118 INFO    HandlerThread:12305 [interfaces.py:finish():202] Joined gpu monitor
+2024-08-23 16:46:46,118 INFO    HandlerThread:12305 [interfaces.py:finish():202] Joined memory monitor
+2024-08-23 16:46:46,118 INFO    HandlerThread:12305 [interfaces.py:finish():202] Joined network monitor
+2024-08-23 16:46:46,118 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:46,119 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 2
+2024-08-23 16:46:46,119 INFO    SenderThread:12305 [sender.py:transition_state():617] send defer: 3
+2024-08-23 16:46:46,119 DEBUG   SenderThread:12305 [sender.py:send():382] send: stats
+2024-08-23 16:46:46,119 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:46,119 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 3
+2024-08-23 16:46:46,120 DEBUG   SenderThread:12305 [sender.py:send():382] send: history
+2024-08-23 16:46:46,121 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:46:46,122 INFO    SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:46:46,122 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:46,122 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 3
+2024-08-23 16:46:46,122 INFO    SenderThread:12305 [sender.py:transition_state():617] send defer: 4
+2024-08-23 16:46:46,122 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:46,122 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 4
+2024-08-23 16:46:46,122 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:46,122 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 4
+2024-08-23 16:46:46,122 INFO    SenderThread:12305 [sender.py:transition_state():617] send defer: 5
+2024-08-23 16:46:46,122 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:46,122 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 5
+2024-08-23 16:46:46,123 DEBUG   SenderThread:12305 [sender.py:send():382] send: summary
+2024-08-23 16:46:46,124 INFO    SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:46:46,124 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:46,124 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 5
+2024-08-23 16:46:46,124 INFO    SenderThread:12305 [sender.py:transition_state():617] send defer: 6
+2024-08-23 16:46:46,124 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:46,124 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 6
+2024-08-23 16:46:46,125 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:46,125 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 6
+2024-08-23 16:46:46,125 INFO    SenderThread:12305 [sender.py:transition_state():617] send defer: 7
+2024-08-23 16:46:46,125 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:46:46,125 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:46,125 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 7
+2024-08-23 16:46:46,125 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:46,125 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 7
+2024-08-23 16:46:47,040 INFO    Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json
+2024-08-23 16:46:47,081 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 16:46:47,096 INFO    SenderThread:12305 [sender.py:transition_state():617] send defer: 8
+2024-08-23 16:46:47,096 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 16:46:47,096 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:47,097 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 8
+2024-08-23 16:46:47,097 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:47,097 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 8
+2024-08-23 16:46:47,097 INFO    SenderThread:12305 [job_builder.py:build():296] Attempting to build job artifact
+2024-08-23 16:46:47,098 INFO    SenderThread:12305 [job_builder.py:_get_source_type():426] is repo sourced job
+2024-08-23 16:46:47,113 INFO    SenderThread:12305 [job_builder.py:build():402] adding wandb-job metadata file
+2024-08-23 16:46:47,121 INFO    SenderThread:12305 [sender.py:transition_state():617] send defer: 9
+2024-08-23 16:46:47,122 DEBUG   SenderThread:12305 [sender.py:send():382] send: artifact
+2024-08-23 16:46:47,122 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:47,123 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 9
+2024-08-23 16:46:48,001 INFO    SenderThread:12305 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE2MjAxODA1Mw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjQ1ODQ1MA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE2MjAxODA1Mw==', 'versionIndex': 3}}}
+2024-08-23 16:46:48,001 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:48,001 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 9
+2024-08-23 16:46:48,001 INFO    SenderThread:12305 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-08-23 16:46:48,041 INFO    SenderThread:12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:46:48,041 INFO    SenderThread:12305 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240823_163849-faey1t8u/files
+2024-08-23 16:46:48,042 INFO    SenderThread:12305 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_163849-faey1t8u/files/requirements.txt requirements.txt
+2024-08-23 16:46:48,042 INFO    SenderThread:12305 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_163849-faey1t8u/files/config.yaml config.yaml
+2024-08-23 16:46:48,043 INFO    SenderThread:12305 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-metadata.json wandb-metadata.json
+2024-08-23 16:46:48,043 INFO    SenderThread:12305 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json wandb-summary.json
+2024-08-23 16:46:48,045 INFO    SenderThread:12305 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_163849-faey1t8u/files/output.log output.log
+2024-08-23 16:46:48,046 INFO    SenderThread:12305 [sender.py:transition_state():617] send defer: 10
+2024-08-23 16:46:48,047 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:48,048 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 10
+2024-08-23 16:46:48,048 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:48,048 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 10
+2024-08-23 16:46:48,048 INFO    SenderThread:12305 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-23 16:46:48,082 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 16:46:48,082 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 16:46:49,082 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 16:46:49,082 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 16:46:50,083 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 16:46:50,083 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 16:46:50,119 INFO    wandb-upload_2:12305 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json
+2024-08-23 16:46:50,148 INFO    wandb-upload_3:12305 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_163849-faey1t8u/files/output.log
+2024-08-23 16:46:50,166 INFO    wandb-upload_0:12305 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_163849-faey1t8u/files/requirements.txt
+2024-08-23 16:46:50,204 INFO    wandb-upload_1:12305 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_163849-faey1t8u/files/config.yaml
+2024-08-23 16:46:50,405 INFO    Thread-11 (_thread_body):12305 [sender.py:transition_state():617] send defer: 11
+2024-08-23 16:46:50,405 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:50,405 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 11
+2024-08-23 16:46:50,405 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:50,405 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 11
+2024-08-23 16:46:50,405 INFO    SenderThread:12305 [file_pusher.py:join():178] waiting for file pusher
+2024-08-23 16:46:50,405 INFO    SenderThread:12305 [sender.py:transition_state():617] send defer: 12
+2024-08-23 16:46:50,405 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:50,406 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 12
+2024-08-23 16:46:50,406 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:50,406 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 12
+2024-08-23 16:46:50,406 INFO    SenderThread:12305 [file_stream.py:finish():595] file stream finish called
+2024-08-23 16:46:51,083 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 16:46:51,097 INFO    SenderThread:12305 [file_stream.py:finish():599] file stream finish is done
+2024-08-23 16:46:51,097 INFO    SenderThread:12305 [sender.py:transition_state():617] send defer: 13
+2024-08-23 16:46:51,097 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 16:46:51,097 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:51,098 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 13
+2024-08-23 16:46:51,098 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:51,098 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 13
+2024-08-23 16:46:51,098 INFO    SenderThread:12305 [sender.py:transition_state():617] send defer: 14
+2024-08-23 16:46:51,098 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:46:51,098 DEBUG   SenderThread:12305 [sender.py:send():382] send: final
+2024-08-23 16:46:51,098 INFO    HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 14
+2024-08-23 16:46:51,098 DEBUG   SenderThread:12305 [sender.py:send():382] send: footer
+2024-08-23 16:46:51,098 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:46:51,099 INFO    SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 14
+2024-08-23 16:46:51,099 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 16:46:51,099 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 16:46:51,099 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 16:46:51,100 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: server_info
+2024-08-23 16:46:51,100 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 16:46:51,100 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: get_summary
+2024-08-23 16:46:51,100 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: server_info
+2024-08-23 16:46:51,101 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: sampled_history
+2024-08-23 16:46:51,103 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:46:51,103 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: job_info
+2024-08-23 16:46:51,261 DEBUG   SenderThread:12305 [sender.py:send_request():409] send_request: job_info
+2024-08-23 16:46:51,262 INFO    MainThread:12305 [wandb_run.py:_footer_history_summary_info():3866] rendering history
+2024-08-23 16:46:51,262 INFO    MainThread:12305 [wandb_run.py:_footer_history_summary_info():3898] rendering summary
+2024-08-23 16:46:51,262 INFO    MainThread:12305 [wandb_run.py:_footer_sync_info():3825] logging synced files
+2024-08-23 16:46:51,263 DEBUG   HandlerThread:12305 [handler.py:handle_request():146] handle_request: shutdown
+2024-08-23 16:46:51,263 INFO    HandlerThread:12305 [handler.py:finish():869] shutting down handler
+2024-08-23 16:46:52,103 INFO    WriterThread:12305 [datastore.py:close():296] close: /project/wandb/run-20240823_163849-faey1t8u/run-faey1t8u.wandb
+2024-08-23 16:46:52,262 INFO    SenderThread:12305 [sender.py:finish():1572] shutting down sender
+2024-08-23 16:46:52,262 INFO    SenderThread:12305 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-23 16:46:52,262 INFO    SenderThread:12305 [file_pusher.py:join():178] waiting for file pusher

wandb/run-20240823_163849-faey1t8u/logs/debug.log

@@ -0,0 +1,30 @@
+2024-08-23 16:38:49,358 INFO    MainThread:12234 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
+2024-08-23 16:38:49,358 INFO    MainThread:12234 [wandb_setup.py:_flush():76] Configure stats pid to 12234
+2024-08-23 16:38:49,358 INFO    MainThread:12234 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings
+2024-08-23 16:38:49,358 INFO    MainThread:12234 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings
+2024-08-23 16:38:49,359 INFO    MainThread:12234 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'}
+2024-08-23 16:38:49,359 INFO    MainThread:12234 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-08-23 16:38:49,359 INFO    MainThread:12234 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'}
+2024-08-23 16:38:49,359 INFO    MainThread:12234 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240823_163849-faey1t8u/logs/debug.log
+2024-08-23 16:38:49,359 INFO    MainThread:12234 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240823_163849-faey1t8u/logs/debug-internal.log
+2024-08-23 16:38:49,359 INFO    MainThread:12234 [wandb_init.py:init():566] calling init triggers
+2024-08-23 16:38:49,359 INFO    MainThread:12234 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
+config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'test_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 4096, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'Qwen2-0.5b-0.2_train_2024-08-23-16:38:35', 'wandb_project': 'llm_tutorial-0.2', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'save': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 10, 'save_interval': 10, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 7500, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 7500, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 3, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/Qwen2-0.5b-0.2', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 106}
+2024-08-23 16:38:49,359 INFO    MainThread:12234 [wandb_init.py:init():616] starting backend
+2024-08-23 16:38:49,359 INFO    MainThread:12234 [wandb_init.py:init():620] setting up manager
+2024-08-23 16:38:49,364 INFO    MainThread:12234 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-08-23 16:38:49,364 INFO    MainThread:12234 [wandb_init.py:init():628] backend started and connected
+2024-08-23 16:38:49,369 INFO    MainThread:12234 [wandb_init.py:init():720] updated telemetry
+2024-08-23 16:38:49,401 INFO    MainThread:12234 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-08-23 16:38:49,805 INFO    MainThread:12234 [wandb_run.py:_on_init():2262] communicating current version
+2024-08-23 16:38:49,830 INFO    MainThread:12234 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-08-23 16:38:49,830 INFO    MainThread:12234 [wandb_init.py:init():804] starting run threads in backend
+2024-08-23 16:38:49,926 INFO    MainThread:12234 [wandb_run.py:_console_start():2241] atexit reg
+2024-08-23 16:38:49,926 INFO    MainThread:12234 [wandb_run.py:_redirect():2096] redirect: wrap_raw
+2024-08-23 16:38:49,926 INFO    MainThread:12234 [wandb_run.py:_redirect():2161] Wrapping output streams.
+2024-08-23 16:38:49,927 INFO    MainThread:12234 [wandb_run.py:_redirect():2186] Redirects installed.
+2024-08-23 16:38:49,927 INFO    MainThread:12234 [wandb_init.py:init():847] run started, returning control to user process
+2024-08-23 16:38:58,351 INFO    MainThread:12234 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 4096, 'num_attention_heads': 14, 'num_hidden_layers': 24}
+2024-08-23 16:38:58,352 INFO    MainThread:12234 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1}
+2024-08-23 16:46:52,263 WARNING MsgRouterThr:12234 [router.py:message_loop():77] message_loop has been closed

wandb/run-20240823_202540-om09pls8/files/config.yaml

@@ -0,0 +1,342 @@
+wandb_version: 1
+
+sharding_strategy:
+  desc: null
+  value: FULL_SHARD
+checkpoint_type:
+  desc: null
+  value: LOCAL_STATE_DICT
+fsdp_activation_checkpointing:
+  desc: null
+  value: true
+fsdp_cpu_offload:
+  desc: null
+  value: false
+low_cpu_fsdp:
+  desc: null
+  value: false
+no_meta_device:
+  desc: null
+  value: false
+data_path:
+  desc: null
+  value: null
+split:
+  desc: null
+  value: 969, 30, 1
+train_data_path:
+  desc: null
+  value:
+  - '1754785366'
+  - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document
+  - '28623823675'
+  - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document
+valid_data_path:
+  desc: null
+  value:
+  - '1754785366'
+  - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document
+test_data_path:
+  desc: null
+  value:
+  - '1754785366'
+  - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document
+data_cache_path:
+  desc: null
+  value: null
+vocab_size:
+  desc: null
+  value: null
+vocab_file:
+  desc: null
+  value: null
+merge_file:
+  desc: null
+  value: null
+seq_length:
+  desc: null
+  value: 1024
+num_workers:
+  desc: null
+  value: 2
+tokenizer_type:
+  desc: null
+  value: HFPreTrainedTokenizer
+tokenizer_model:
+  desc: null
+  value: /share/pretrained_lm/Qwen/Qwen2-0.5B
+reset_position_ids:
+  desc: null
+  value: false
+reset_attention_mask:
+  desc: null
+  value: false
+eod_mask_loss:
+  desc: null
+  value: false
+retro_return_doc_ids:
+  desc: null
+  value: false
+short_seq_prob:
+  desc: null
+  value: 0.1
+vocab_extra_ids:
+  desc: null
+  value: 0
+seed:
+  desc: null
+  value: 1234
+use_mpi:
+  desc: null
+  value: false
+wandb_entity:
+  desc: null
+  value: iwakawa-koichi-q5-tohoku-nlp6723
+wandb_name:
+  desc: null
+  value: Qwen2-0.5b-0.2_train_2024-08-23-20:25:00
+wandb_project:
+  desc: null
+  value: llm_tutorial-0.2
+quantization:
+  desc: null
+  value: false
+use_freeze_layers:
+  desc: null
+  value: false
+freeze_layers:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+mixed_precision:
+  desc: null
+  value: true
+param_dtype:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: /work/llm_recipes/models/Qwen2-0.5b-0.2
+save:
+  desc: null
+  value: /work/llm_recipes/models/Qwen2-0.5b-0.2
+base_model:
+  desc: null
+  value: /share/pretrained_lm/Qwen/Qwen2-0.5B
+use_better_transformer:
+  desc: null
+  value: false
+grad_clip_norm:
+  desc: null
+  value: 1.0
+eval_interval:
+  desc: null
+  value: 3
+save_interval:
+  desc: null
+  value: 500
+eval_iters:
+  desc: null
+  value: 10
+optimizer:
+  desc: null
+  value: anyprecision
+lr:
+  desc: null
+  value: 2.0e-05
+lr_decay_style:
+  desc: null
+  value: cosine
+lr_decay_iters:
+  desc: null
+  value: 16000
+lr_warmup_iters:
+  desc: null
+  value: 500
+min_lr:
+  desc: null
+  value: 1.0e-06
+train_iters:
+  desc: null
+  value: 16000
+train_samples:
+  desc: null
+  value: null
+global_batch_size:
+  desc: null
+  value: 612
+micro_batch_size:
+  desc: null
+  value: 17
+make_vocab_size_divisible_by:
+  desc: null
+  value: 128
+sliding_window_size:
+  desc: null
+  value: 131072
+skip_batch:
+  desc: null
+  value: null
+no_save_optimizer_state:
+  desc: null
+  value: false
+continual_pretraining:
+  desc: null
+  value: false
+instruction_tuning:
+  desc: null
+  value: false
+direct_preference_optimization:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.1
+hidden_dropout:
+  desc: null
+  value: 0.1
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_eps:
+  desc: null
+  value: 1.0e-06
+hf_transformer_model_dir:
+  desc: null
+  value: null
+instruction_train_data_path:
+  desc: null
+  value: null
+instruction_valid_data_path:
+  desc: null
+  value: null
+epoch:
+  desc: null
+  value: null
+instruction_dataset_size:
+  desc: null
+  value: null
+save_sampler_state:
+  desc: null
+  value: false
+label_smoothing:
+  desc: null
+  value: 0.0
+save_n_checkpoints:
+  desc: null
+  value: 10
+hf_repo_id:
+  desc: null
+  value: koichi12/Qwen2-0.5b-0.2
+create_public_hf_repo:
+  desc: null
+  value: false
+upload_all_checkpoints_to_hf:
+  desc: null
+  value: true
+hf_upload_retry_limit:
+  desc: null
+  value: 2
+exit_duration_in_mins:
+  desc: null
+  value: null
+source_key:
+  desc: null
+  value: null
+target_key:
+  desc: null
+  value: null
+attn_implementation:
+  desc: null
+  value: flash_attention_2
+efficient_instruction_tuning:
+  desc: null
+  value: false
+remove_padding_masking:
+  desc: null
+  value: false
+save_start_iter:
+  desc: null
+  value: null
+valid_micro_batch_size:
+  desc: null
+  value: 10
+rank:
+  desc: null
+  value: 0
+world_size:
+  desc: null
+  value: 4
+padded_vocab_size:
+  desc: null
+  value: 151680
+gradient_accumulation_steps:
+  desc: null
+  value: 9
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.43.3
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1724412340.7504
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      - 105
+      2:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      - 105
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.43.3
+      8:
+      - 5
+      13: linux-x86_64
+model_architecture:
+  desc: null
+  value: Qwen2ForCausalLM
+activation_function:
+  desc: null
+  value: silu
+hidden_size:
+  desc: null
+  value: 896
+model_type:
+  desc: null
+  value: qwen2
+max_position_embeddings:
+  desc: null
+  value: 1024
+num_attention_heads:
+  desc: null
+  value: 14
+num_hidden_layers:
+  desc: null
+  value: 24

wandb/run-20240823_202540-om09pls8/files/output.log

@@ -0,0 +1,133 @@
+Created Hugging Face repository with ID koichi12/Qwen2-0.5b-0.2.
+Clearing GPU cache for all ranks
+--> Running with torch torch_distributed debug set to detail
+Loading model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000100/model.pt
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+Loaded model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000100/model.pt
+--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B
+--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params
+BFloat16 enabled for mixed precision - using bfSixteen policy
+--> applying fsdp activation checkpointing...
+ > datasets target sizes (minimum size):
+    train:      9792000
+    validation: 32644080
+    test:       6120
+> building train, validation, and test datasets for GPT ...
+Let split = None
+Unable to save the indexes because path_to_cache is None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+> finished creating GPT datasets ...
+Loading optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000100/optimizer.pt
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+[rank0]:[2024-08-23 20:26:02,460] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling:  defaultdict(<class 'float'>, {})
+Loaded optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000100/optimizer.pt
+model info: FullyShardedDataParallel(
+  (_fsdp_wrapped_module): Qwen2ForCausalLM(
+    (model): Qwen2Model(
+      (embed_tokens): Embedding(151936, 896)
+      (layers): ModuleList(
+        (0-23): 24 x FullyShardedDataParallel(
+          (_fsdp_wrapped_module): CheckpointWrapper(
+            (_checkpoint_wrapped_module): Qwen2DecoderLayer(
+              (self_attn): Qwen2FlashAttention2(
+                (q_proj): Linear(in_features=896, out_features=896, bias=True)
+                (k_proj): Linear(in_features=896, out_features=128, bias=True)
+                (v_proj): Linear(in_features=896, out_features=128, bias=True)
+                (o_proj): Linear(in_features=896, out_features=896, bias=False)
+                (rotary_emb): Qwen2RotaryEmbedding()
+              )
+              (mlp): Qwen2MLP(
+                (gate_proj): Linear(in_features=896, out_features=4864, bias=False)
+                (up_proj): Linear(in_features=896, out_features=4864, bias=False)
+                (down_proj): Linear(in_features=4864, out_features=896, bias=False)
+                (act_fn): SiLU()
+              )
+              (input_layernorm): Qwen2RMSNorm()
+              (post_attention_layernorm): Qwen2RMSNorm()
+            )
+          )
+        )
+      )
+      (norm): Qwen2RMSNorm()
+    )
+    (lm_head): Linear(in_features=896, out_features=151936, bias=False)
+  )
+)
+model config: Qwen2Config {
+  "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "label_smoothing": 0.0,
+  "max_position_embeddings": 1024,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.3",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}
+------------------------------------------------------------------
+iteration: 101 , TFLOPS: 23.617124223738323, Tokens per sec: 29262.917733714043, Loss: 3.7116615772247314
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 102 , TFLOPS: 72.97505903558803, Tokens per sec: 90420.11757828314, Loss: 3.7358791828155518
+------------------------------------------------------------------
+ eval ppl=30.41041374206543, eval loss=3.414785146713257
+------------------------------------------------------------------
+iteration: 103 , TFLOPS: 65.5275483737983, Tokens per sec: 81192.2416628126, Loss: 3.757955551147461
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 104 , TFLOPS: 73.41964709398604, Tokens per sec: 90970.9866703471, Loss: 3.730485439300537
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 105 , TFLOPS: 70.30393549004248, Tokens per sec: 87110.44838107748, Loss: 3.7091140747070312
+------------------------------------------------------------------
+ eval ppl=30.61298179626465, eval loss=3.421424150466919
+------------------------------------------------------------------
+iteration: 106 , TFLOPS: 67.02992418712539, Tokens per sec: 83053.76804570397, Loss: 3.732792377471924
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 107 , TFLOPS: 72.30676298983876, Tokens per sec: 89592.06196815638, Loss: 3.7457761764526367
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 108 , TFLOPS: 73.24548732883933, Tokens per sec: 90755.19312868938, Loss: 3.7291133403778076
+------------------------------------------------------------------
+ eval ppl=31.11376953125, eval loss=3.437650442123413
+------------------------------------------------------------------
+iteration: 109 , TFLOPS: 66.08722155273428, Tokens per sec: 81885.7075580594, Loss: 3.71726131439209
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 110 , TFLOPS: 73.33268637131981, Tokens per sec: 90863.23754520644, Loss: 3.7274892330169678
+------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/project/examples/finetuning.py", line 13, in <module>
+    main()
+  File "/project/src/llama_recipes/finetuning.py", line 282, in main
+    train(
+  File "/project/src/llama_recipes/utils/train_utils.py", line 118, in train
+    loss.backward()
+  File "/usr/local/lib/python3.10/dist-packages/torch/_tensor.py", line 522, in backward
+    torch.autograd.backward(
+  File "/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py", line 267, in backward
+    _engine_run_backward(
+  File "/usr/local/lib/python3.10/dist-packages/torch/autograd/graph.py", line 681, in _engine_run_backward
+    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+KeyboardInterrupt

wandb/run-20240823_202540-om09pls8/files/requirements.txt

@@ -0,0 +1,375 @@
+absl-py==2.1.0
+accelerate==0.23.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+apex==0.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+astroid==3.2.4
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+beautifulsoup4==4.12.3
+bert-score==0.3.13
+bleach==6.1.0
+blis==0.7.11
+build==1.2.1
+cachecontrol==0.14.0
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+cleo==2.1.0
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+cmake==3.28.1
+colorama==0.4.6
+comm==0.2.1
+confection==0.1.4
+contourpy==1.2.0
+cramjam==2.8.3
+crashtest==0.4.1
+cryptography==43.0.0
+cubinlinker==0.3.0+2.g405ac64
+cuda-python==12.3.0rc4+9.gdb8c48a.dirty
+cudf==23.12.0
+cugraph-dgl==23.12.0
+cugraph-service-client==23.12.0
+cugraph-service-server==23.12.0
+cugraph==23.12.0
+cuml==23.12.0
+cupy-cuda12x==12.3.0
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.8
+dask-cuda==23.12.0
+dask-cudf==23.12.0
+dask==2023.11.0
+dataclasses-json==0.6.7
+dataproperty==1.0.1
+datasets==2.20.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.8
+distlib==0.3.8
+distributed==2023.11.0
+distro==1.9.0
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+dulwich==0.21.7
+einops==0.7.0
+emoji==2.12.1
+entmax==1.3
+evaluate==0.4.2
+exceptiongroup==1.2.0
+execnet==2.0.2
+executing==2.0.1
+expecttest==0.1.3
+fastjsonschema==2.19.1
+fastparquet==2023.10.1
+fastrlock==0.8.2
+filelock==3.13.1
+flash-attn==2.4.2
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2023.12.2
+fugashi==1.3.2
+fuzzywuzzy==0.18.0
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.27.0
+graphsurgeon==0.4.6
+greenlet==3.0.3
+grpcio==1.60.1
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.24.5
+hydra-core==1.3.2
+hypothesis==5.35.1
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+installer==0.7.0
+intel-openmp==2021.4.0
+ipadic==1.0.0
+ipykernel==6.29.2
+ipython-genutils==0.2.0
+ipython==8.21.0
+isort==5.13.2
+jaraco.classes==3.4.0
+jedi==0.19.1
+jeepney==0.8.0
+jinja2==3.1.3
+jiter==0.5.0
+joblib==1.3.2
+json5==0.9.14
+jsonargparse==3.13.1
+jsonlines==4.0.0
+jsonnet==0.19.1
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema-specifications==2023.12.1
+jsonschema==4.21.1
+jupyter-client==8.6.0
+jupyter-core==5.7.1
+jupyter-tensorboard==0.2.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==1.2.0
+jupyterlab==2.3.2
+jupytext==1.16.1
+keyring==24.3.1
+kiwisolver==1.4.5
+langchain-community==0.2.12
+langchain-core==0.2.31
+langchain-huggingface==0.0.2
+langchain-openai==0.1.21
+langchain-text-splitters==0.2.2
+langchain==0.2.13
+langcodes==3.3.0
+langsmith==0.1.99
+lazy-loader==0.3
+levenshtein==0.25.1
+librosa==0.10.1
+lightning-utilities==0.11.6
+llm-jp-eval==1.4.0
+llvmlite==0.40.1
+lm-eval==0.3.0
+locket==1.0.0
+logzero==1.7.0
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown==3.5.2
+markupsafe==2.1.4
+marshmallow==3.21.3
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mbstrdecoder==1.1.3
+mccabe==0.7.0
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mecab-python3==1.0.6
+mistune==3.0.2
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mkl==2021.1.1
+mock==5.1.0
+mojimoji==0.0.13
+more-itertools==9.1.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+multiprocess==0.70.16
+murmurhash==1.0.10
+mypy-extensions==1.0.0
+nbclient==0.9.0
+nbconvert==7.16.0
+nbformat==5.9.2
+neologdn==0.5.3
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.4.10
+numba==0.57.1+1.g1ff679645
+numexpr==2.10.1
+numpy==1.24.4
+nvfuser==0.1.4a0+d0bb811
+nvidia-dali-cuda120==1.34.0
+nvidia-pyindex==1.0.9
+nvtx==0.2.5
+oauthlib==3.2.2
+omegaconf==2.3.0
+onnx==1.15.0rc2
+openai==1.40.6
+opencv==4.7.0
+optree==0.10.0
+orjson==3.10.7
+packaging==23.2
+pandas==2.2.2
+pandocfilters==1.5.1
+parso==0.8.3
+partd==1.4.1
+pathvalidate==3.2.0
+peft==0.5.0
+pexpect==4.9.0
+pillow==10.2.0
+pip==24.0
+pkginfo==1.11.1
+plac==1.4.3
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+poetry-core==1.9.0
+poetry-plugin-export==1.8.0
+poetry==1.8.3
+polygraphy==0.49.4
+pooch==1.8.0
+portalocker==2.10.1
+preshed==3.0.9
+prettytable==3.9.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.24.4
+psutil==5.9.4
+ptxcompiler==0.8.1+2.g0d406d6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow-hotfix==0.6
+pyarrow==15.0.2
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pybind11-global==2.11.1
+pybind11==2.11.1
+pycocotools==2.0+nv0.8.0
+pycountry==24.6.1
+pycparser==2.21
+pydantic-core==2.16.2
+pydantic==2.6.1
+pygments==2.17.2
+pylibcugraph==23.12.0
+pylibcugraphops==23.12.0
+pylibraft==23.12.0
+pylint==3.2.6
+pynvml==11.4.1
+pyparsing==3.1.1
+pyproject-hooks==1.1.0
+pytablewriter==1.2.0
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==13.0
+pytest-shard==0.1.2
+pytest-xdist==3.5.0
+pytest==8.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-hostlist==1.23.0
+python-levenshtein==0.25.1
+pytorch-lightning==2.4.0
+pytorch-quantization==2.1.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.2
+raft-dask==23.12.0
+rapidfuzz==3.9.6
+rapids-dask-dependency==23.12.1
+referencing==0.33.0
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests-toolbelt==1.0.0
+requests==2.32.3
+rhoknp==1.7.0
+rich==13.7.0
+rmm==23.12.0
+rouge-score==0.1.2
+rpds-py==0.17.1
+rsa==4.9
+sacrebleu==2.4.2
+safetensors==0.4.3
+scikit-learn==1.5.1
+scipy==1.12.0
+secretstorage==3.3.3
+send2trash==1.8.2
+sentence-transformers==3.0.1
+sentencepiece==0.1.99
+sentry-sdk==2.12.0
+setproctitle==1.3.3
+setuptools==68.2.2
+shellingham==1.5.4
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sniffio==1.3.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sphinx-glpi-theme==0.6
+sqlalchemy==2.0.32
+sqlitedict==2.1.0
+srsly==2.4.8
+stack-data==0.6.3
+sumeval==0.2.2
+sympy==1.12
+tabledata==1.3.3
+tabulate==0.9.0
+tbb==2021.11.0
+tblib==3.0.0
+tcolorpy==0.1.6
+tenacity==8.5.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.9.0
+tensorrt==8.6.3
+terminado==0.18.0
+termplotlib==0.3.9
+text-generation==0.7.0
+thinc==8.2.3
+threadpoolctl==3.2.0
+thriftpy2==0.4.17
+tiktoken==0.7.0
+tinycss2==1.2.1
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+tomlkit==0.13.2
+toolz==0.12.1
+torch-tensorrt==2.3.0a0
+torch==2.3.0a0+ebedce2
+torchdata==0.7.1a0
+torchmetrics==0.10.3
+torchtext==0.17.0a0
+torchvision==0.18.0a0
+tornado==6.4
+tqdm-multiprocess==0.0.11
+tqdm==4.66.5
+traitlets==5.9.0
+transformer-engine==1.3.0+5b90b7f
+transformers==4.43.3
+treelite-runtime==3.9.1
+treelite==3.9.1
+triton==2.2.0+e28a256
+trove-classifiers==2024.7.2
+typepy==1.3.2
+typer==0.9.0
+types-dataclasses==0.6.6
+typing-extensions==4.12.2
+typing-inspect==0.9.0
+tzdata==2024.1
+ucx-py==0.35.0
+uff==0.6.9
+ujson==5.8.0
+unbabel-comet==2.2.2
+unidic-lite==1.0.8
+urllib3==1.26.18
+virtualenv==20.26.3
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.3.4
+webencodings==0.5.1
+werkzeug==3.0.1
+wheel==0.42.0
+word2number==1.1
+xdoctest==1.0.2
+xgboost==1.7.6
+xmltodict==0.13.0
+xxhash==3.4.1
+yarl==1.9.4
+zict==3.0.0
+zipp==3.17.0
+zstandard==0.23.0

wandb/run-20240823_202540-om09pls8/files/wandb-metadata.json

@@ -0,0 +1,502 @@
+{
+    "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-08-23T11:25:41.454442",
+    "startedAt": "2024-08-23T11:25:40.735970",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--seq-length",
+        "1024",
+        "--sliding-window-size",
+        "131072",
+        "--micro-batch-size",
+        "17",
+        "--valid_micro_batch_size",
+        "10",
+        "--global-batch-size",
+        "612",
+        "--train-iters",
+        "16000",
+        "--tokenizer-type",
+        "HFPreTrainedTokenizer",
+        "--tokenizer-model",
+        "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+        "--train-data-path",
+        "1754785366",
+        "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document",
+        "28623823675",
+        "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document",
+        "--valid-data-path",
+        "1754785366",
+        "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document",
+        "--test-data-path",
+        "1754785366",
+        "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document",
+        "--lr",
+        "2e-5",
+        "--min-lr",
+        "1e-6",
+        "--lr-decay-style",
+        "cosine",
+        "--lr-warmup-iters",
+        "500",
+        "--lr-decay-iters",
+        "16000",
+        "--weight-decay",
+        "0.1",
+        "--grad-clip-norm",
+        "1.0",
+        "--optimizer",
+        "anyprecision",
+        "--adam-beta1",
+        "0.9",
+        "--adam-beta2",
+        "0.95",
+        "--adam-eps",
+        "1e-6",
+        "--save-interval",
+        "500",
+        "--eval-interval",
+        "3",
+        "--eval-iters",
+        "10",
+        "--bf16",
+        "--mixed-precision",
+        "--base-model",
+        "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+        "--save",
+        "/work/llm_recipes/models/Qwen2-0.5b-0.2",
+        "--load",
+        "/work/llm_recipes/models/Qwen2-0.5b-0.2",
+        "--fsdp-activation-checkpointing",
+        "--sharding-strategy",
+        "FULL_SHARD",
+        "--checkpoint-type",
+        "LOCAL_STATE_DICT",
+        "--save-n-checkpoints",
+        "10",
+        "--upload-all-checkpoints-to-hf",
+        "--hf-upload-retry-limit",
+        "2",
+        "--hf-repo-id",
+        "koichi12/Qwen2-0.5b-0.2",
+        "--wandb-entity",
+        "iwakawa-koichi-q5-tohoku-nlp6723",
+        "--wandb-project",
+        "llm_tutorial-0.2",
+        "--wandb-name",
+        "Qwen2-0.5b-0.2_train_2024-08-23-20:25:00"
+    ],
+    "state": "running",
+    "program": "/project/examples/finetuning.py",
+    "codePathLocal": "examples/finetuning.py",
+    "codePath": "examples/finetuning.py",
+    "git": {
+        "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git",
+        "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6"
+    },
+    "email": null,
+    "root": "/project",
+    "host": "gpu-koiwa-00",
+    "username": "koiwa",
+    "executable": "/usr/bin/python",
+    "cpu_count": 72,
+    "cpu_count_logical": 72,
+    "cpu_freq": {
+        "current": 2400.038999999999,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 0.0625,
+            "used": 1.1444091796875e-05
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 4,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        },
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        },
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        },
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 226.66352462768555
+    }
+}

wandb/run-20240823_202540-om09pls8/files/wandb-summary.json

@@ -0,0 +1 @@
+{"training/loss": 3.7274892330169678, "training/perplexity": 41.57459289701456, "utils/batch_size": 17, "utils/global_batch_size": 612, "utils/seq_len": 1025, "utils/gradient_accumulation_steps": 9, "utils/iteration": 110, "optimizer/lr": 5.18e-06, "optimizer/variance_l2": 0.04658828859035772, "optimizer/variance_sqrt_l2": 0.8728927373830674, "optimizer/momentum_l2": 0.8215464439288661, "optimizer/weight_l2": 640.8711356427281, "optimizer/variance_l1": 0.7605819702148438, "optimizer/variance_sqrt_l1": 1761.25, "optimizer/momentum_l1": 1269.75, "optimizer/weight_l1": 1809664.0, "optimizer/variance_abs_max": 0.04248046875, "optimizer/variance_sqrt_abs_max": 0.2060546875, "optimizer/momentum_abs_max": 0.18359375, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 6.903782178000256, "stats/tokens_per_sec": 90863.23754520644, "stats/tokens_per_sec_per_gpu": 22715.80938630161, "stats/tflops": 73.33268637131981, "_timestamp": 1724412471.451136, "_runtime": 130.7007360458374, "_step": 110, "evaluation/val_loss": 3.437650442123413, "evaluation/val_ppl": 31.11376953125, "_wandb": {"runtime": 131}}

wandb/run-20240823_202540-om09pls8/logs/debug-internal.log

@@ -0,0 +1,312 @@
+2024-08-23 20:25:40,751 INFO    StreamThr :13176 [internal.py:wandb_internal():86] W&B internal server running at pid: 13176, started at: 2024-08-23 20:25:40.750453
+2024-08-23 20:25:40,752 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status
+2024-08-23 20:25:40,755 INFO    WriterThread:13176 [datastore.py:open_for_write():87] open: /project/wandb/run-20240823_202540-om09pls8/run-om09pls8.wandb
+2024-08-23 20:25:40,756 DEBUG   SenderThread:13176 [sender.py:send():382] send: header
+2024-08-23 20:25:40,869 DEBUG   SenderThread:13176 [sender.py:send():382] send: run
+2024-08-23 20:25:41,336 INFO    SenderThread:13176 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240823_202540-om09pls8/files
+2024-08-23 20:25:41,336 INFO    SenderThread:13176 [sender.py:_start_run_threads():1136] run started: om09pls8 with start time 1724412340.7504
+2024-08-23 20:25:41,342 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: check_version
+2024-08-23 20:25:41,342 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: check_version
+2024-08-23 20:25:41,412 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: run_start
+2024-08-23 20:25:41,418 DEBUG   HandlerThread:13176 [system_info.py:__init__():27] System info init
+2024-08-23 20:25:41,418 DEBUG   HandlerThread:13176 [system_info.py:__init__():42] System info init done
+2024-08-23 20:25:41,418 INFO    HandlerThread:13176 [system_monitor.py:start():194] Starting system monitor
+2024-08-23 20:25:41,418 INFO    SystemMonitor:13176 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-08-23 20:25:41,418 INFO    HandlerThread:13176 [system_monitor.py:probe():214] Collecting system info
+2024-08-23 20:25:41,419 INFO    SystemMonitor:13176 [interfaces.py:start():190] Started cpu monitoring
+2024-08-23 20:25:41,419 INFO    SystemMonitor:13176 [interfaces.py:start():190] Started disk monitoring
+2024-08-23 20:25:41,420 INFO    SystemMonitor:13176 [interfaces.py:start():190] Started gpu monitoring
+2024-08-23 20:25:41,421 INFO    SystemMonitor:13176 [interfaces.py:start():190] Started memory monitoring
+2024-08-23 20:25:41,423 INFO    SystemMonitor:13176 [interfaces.py:start():190] Started network monitoring
+2024-08-23 20:25:41,454 DEBUG   HandlerThread:13176 [system_info.py:probe():151] Probing system
+2024-08-23 20:25:41,456 DEBUG   HandlerThread:13176 [system_info.py:_probe_git():136] Probing git
+2024-08-23 20:25:41,470 DEBUG   HandlerThread:13176 [system_info.py:_probe_git():144] Probing git done
+2024-08-23 20:25:41,470 DEBUG   HandlerThread:13176 [system_info.py:probe():199] Probing system done
+2024-08-23 20:25:41,470 DEBUG   HandlerThread:13176 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-23T11:25:41.454442', 'startedAt': '2024-08-23T11:25:40.735970', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--sliding-window-size', '131072', '--micro-batch-size', '17', '--valid_micro_batch_size', '10', '--global-batch-size', '612', '--train-iters', '16000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document', '--valid-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--test-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '16000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '500', '--eval-interval', '3', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--load', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/Qwen2-0.5b-0.2', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial-0.2', '--wandb-name', 'Qwen2-0.5b-0.2_train_2024-08-23-20:25:00'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '887a2cc5d104c10264701f95cbbb0a6a116768d6'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 72, 'cpu_count_logical': 72, 'cpu_freq': {'current': 2400.038999999999, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 226.66352462768555}}
+2024-08-23 20:25:41,470 INFO    HandlerThread:13176 [system_monitor.py:probe():224] Finished collecting system info
+2024-08-23 20:25:41,470 INFO    HandlerThread:13176 [system_monitor.py:probe():227] Publishing system info
+2024-08-23 20:25:41,472 INFO    HandlerThread:13176 [system_monitor.py:probe():229] Finished publishing system info
+2024-08-23 20:25:41,502 DEBUG   SenderThread:13176 [sender.py:send():382] send: files
+2024-08-23 20:25:41,502 INFO    SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now
+2024-08-23 20:25:41,513 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: python_packages
+2024-08-23 20:25:41,513 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 20:25:41,514 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 20:25:41,514 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: python_packages
+2024-08-23 20:25:41,516 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 20:25:41,704 DEBUG   SenderThread:13176 [sender.py:send():382] send: telemetry
+2024-08-23 20:25:42,229 INFO    wandb-upload_0:13176 [upload_job.py:push():131] Uploaded file /tmp/tmpnyk2zt9mwandb/u7uqpthk-wandb-metadata.json
+2024-08-23 20:25:42,338 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:25:42,339 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_202540-om09pls8/files/wandb-metadata.json
+2024-08-23 20:25:42,339 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_202540-om09pls8/files/requirements.txt
+2024-08-23 20:25:44,339 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:25:46,083 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:25:46,340 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:25:47,341 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:25:51,887 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:25:52,344 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:25:53,345 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:25:55,346 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:25:56,513 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 20:25:56,513 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 20:25:56,513 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 20:25:57,347 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:25:57,765 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:25:58,348 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:25:59,349 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:26:02,595 DEBUG   SenderThread:13176 [sender.py:send():382] send: config
+2024-08-23 20:26:02,595 DEBUG   SenderThread:13176 [sender.py:send():382] send: config
+2024-08-23 20:26:03,351 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:26:03,596 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:26:04,352 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:26:08,596 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:26:11,513 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 20:26:11,514 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 20:26:11,514 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 20:26:13,784 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:26:14,359 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/config.yaml
+2024-08-23 20:26:18,968 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:26:23,969 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:26:28,970 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:26:30,283 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 20:26:30,422 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 20:26:30,422 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 20:26:34,601 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:26:39,602 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:26:39,739 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 20:26:41,424 DEBUG   SystemMonitor:13176 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-08-23 20:26:41,427 DEBUG   SenderThread:13176 [sender.py:send():382] send: stats
+2024-08-23 20:26:42,374 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:26:45,083 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 20:26:45,084 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 20:26:45,084 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:26:45,085 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 20:26:46,679 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 20:26:46,682 DEBUG   SenderThread:13176 [sender.py:send():382] send: history
+2024-08-23 20:26:46,682 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 20:26:46,684 INFO    SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 20:26:47,378 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json
+2024-08-23 20:26:48,379 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:26:49,094 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 20:26:50,093 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:26:50,380 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:26:55,094 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:26:56,822 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 20:26:56,825 DEBUG   SenderThread:13176 [sender.py:send():382] send: history
+2024-08-23 20:26:56,826 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 20:26:56,829 INFO    SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 20:26:57,385 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json
+2024-08-23 20:26:58,386 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:27:00,084 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 20:27:00,085 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 20:27:00,085 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 20:27:00,326 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:27:03,720 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 20:27:03,722 DEBUG   SenderThread:13176 [sender.py:send():382] send: history
+2024-08-23 20:27:03,722 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 20:27:03,723 INFO    SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 20:27:04,390 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json
+2024-08-23 20:27:05,764 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:27:06,392 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:27:10,765 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:27:10,923 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 20:27:10,926 DEBUG   SenderThread:13176 [sender.py:send():382] send: history
+2024-08-23 20:27:10,926 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 20:27:10,928 INFO    SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 20:27:11,395 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json
+2024-08-23 20:27:11,429 DEBUG   SenderThread:13176 [sender.py:send():382] send: stats
+2024-08-23 20:27:12,396 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:27:13,173 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 20:27:14,397 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:27:15,084 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 20:27:15,084 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 20:27:15,085 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 20:27:16,306 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:27:20,728 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 20:27:20,730 DEBUG   SenderThread:13176 [sender.py:send():382] send: history
+2024-08-23 20:27:20,730 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 20:27:20,731 INFO    SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 20:27:21,402 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json
+2024-08-23 20:27:21,772 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:27:22,403 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:27:26,773 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:27:27,731 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 20:27:27,734 DEBUG   SenderThread:13176 [sender.py:send():382] send: history
+2024-08-23 20:27:27,734 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 20:27:27,736 INFO    SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 20:27:28,408 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json
+2024-08-23 20:27:30,084 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 20:27:30,085 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 20:27:30,085 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 20:27:30,409 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:27:32,346 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:27:34,646 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 20:27:34,649 DEBUG   SenderThread:13176 [sender.py:send():382] send: history
+2024-08-23 20:27:34,649 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 20:27:34,651 INFO    SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 20:27:35,413 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json
+2024-08-23 20:27:36,413 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:27:36,883 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 20:27:37,883 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:27:38,415 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:27:41,432 DEBUG   SenderThread:13176 [sender.py:send():382] send: stats
+2024-08-23 20:27:43,433 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:27:44,546 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 20:27:44,549 DEBUG   SenderThread:13176 [sender.py:send():382] send: history
+2024-08-23 20:27:44,549 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 20:27:44,551 INFO    SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 20:27:45,085 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 20:27:45,085 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 20:27:45,085 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 20:27:45,420 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json
+2024-08-23 20:27:46,421 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:27:49,351 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:27:51,452 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 20:27:51,454 DEBUG   SenderThread:13176 [sender.py:send():382] send: history
+2024-08-23 20:27:51,454 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 20:27:51,456 INFO    SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 20:27:52,425 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json
+2024-08-23 20:27:53,012 DEBUG   SenderThread:13176 [sender.py:send():382] send: exit
+2024-08-23 20:27:53,012 INFO    SenderThread:13176 [sender.py:send_exit():589] handling exit code: 255
+2024-08-23 20:27:53,013 INFO    SenderThread:13176 [sender.py:send_exit():591] handling runtime: 131
+2024-08-23 20:27:53,014 INFO    SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 20:27:53,014 INFO    SenderThread:13176 [sender.py:send_exit():597] send defer
+2024-08-23 20:27:53,014 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:53,014 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 0
+2024-08-23 20:27:53,015 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:53,015 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 0
+2024-08-23 20:27:53,015 INFO    SenderThread:13176 [sender.py:transition_state():617] send defer: 1
+2024-08-23 20:27:53,015 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:53,015 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 1
+2024-08-23 20:27:53,015 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:53,015 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 1
+2024-08-23 20:27:53,015 INFO    SenderThread:13176 [sender.py:transition_state():617] send defer: 2
+2024-08-23 20:27:53,015 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:53,015 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 2
+2024-08-23 20:27:53,015 INFO    HandlerThread:13176 [system_monitor.py:finish():203] Stopping system monitor
+2024-08-23 20:27:53,015 DEBUG   SystemMonitor:13176 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-08-23 20:27:53,016 DEBUG   SystemMonitor:13176 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-08-23 20:27:53,016 INFO    HandlerThread:13176 [interfaces.py:finish():202] Joined cpu monitor
+2024-08-23 20:27:53,018 INFO    HandlerThread:13176 [interfaces.py:finish():202] Joined disk monitor
+2024-08-23 20:27:53,348 INFO    HandlerThread:13176 [interfaces.py:finish():202] Joined gpu monitor
+2024-08-23 20:27:53,349 INFO    HandlerThread:13176 [interfaces.py:finish():202] Joined memory monitor
+2024-08-23 20:27:53,349 INFO    HandlerThread:13176 [interfaces.py:finish():202] Joined network monitor
+2024-08-23 20:27:53,350 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:53,350 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 2
+2024-08-23 20:27:53,350 INFO    SenderThread:13176 [sender.py:transition_state():617] send defer: 3
+2024-08-23 20:27:53,350 DEBUG   SenderThread:13176 [sender.py:send():382] send: stats
+2024-08-23 20:27:53,350 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:53,351 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 3
+2024-08-23 20:27:53,354 DEBUG   SenderThread:13176 [sender.py:send():382] send: history
+2024-08-23 20:27:53,354 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 20:27:53,355 INFO    SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 20:27:53,355 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:53,355 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 3
+2024-08-23 20:27:53,356 INFO    SenderThread:13176 [sender.py:transition_state():617] send defer: 4
+2024-08-23 20:27:53,356 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:53,356 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 4
+2024-08-23 20:27:53,356 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:53,356 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 4
+2024-08-23 20:27:53,356 INFO    SenderThread:13176 [sender.py:transition_state():617] send defer: 5
+2024-08-23 20:27:53,356 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:53,356 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 5
+2024-08-23 20:27:53,358 DEBUG   SenderThread:13176 [sender.py:send():382] send: summary
+2024-08-23 20:27:53,358 INFO    SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 20:27:53,359 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:53,359 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 5
+2024-08-23 20:27:53,359 INFO    SenderThread:13176 [sender.py:transition_state():617] send defer: 6
+2024-08-23 20:27:53,359 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:53,359 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 6
+2024-08-23 20:27:53,359 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:53,359 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 6
+2024-08-23 20:27:53,359 INFO    SenderThread:13176 [sender.py:transition_state():617] send defer: 7
+2024-08-23 20:27:53,359 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 20:27:53,360 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:53,360 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 7
+2024-08-23 20:27:53,360 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:53,360 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 7
+2024-08-23 20:27:53,427 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json
+2024-08-23 20:27:54,012 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 20:27:54,427 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:27:54,598 INFO    SenderThread:13176 [sender.py:transition_state():617] send defer: 8
+2024-08-23 20:27:54,598 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 20:27:54,598 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:54,598 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 8
+2024-08-23 20:27:54,599 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:54,599 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 8
+2024-08-23 20:27:54,599 INFO    SenderThread:13176 [job_builder.py:build():296] Attempting to build job artifact
+2024-08-23 20:27:54,600 INFO    SenderThread:13176 [job_builder.py:_get_source_type():426] is repo sourced job
+2024-08-23 20:27:54,620 INFO    SenderThread:13176 [job_builder.py:build():402] adding wandb-job metadata file
+2024-08-23 20:27:54,630 INFO    SenderThread:13176 [sender.py:transition_state():617] send defer: 9
+2024-08-23 20:27:54,631 DEBUG   SenderThread:13176 [sender.py:send():382] send: artifact
+2024-08-23 20:27:54,631 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:54,632 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 9
+2024-08-23 20:27:55,013 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 20:27:55,428 INFO    Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:27:55,507 INFO    SenderThread:13176 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE2MTk4ODkxMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjQ1ODQ1MA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE2MjAxODA1Mw==', 'versionIndex': 3}}}
+2024-08-23 20:27:55,507 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:55,507 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 9
+2024-08-23 20:27:55,507 INFO    SenderThread:13176 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-08-23 20:27:56,429 INFO    SenderThread:13176 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240823_202540-om09pls8/files
+2024-08-23 20:27:56,429 INFO    SenderThread:13176 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_202540-om09pls8/files/requirements.txt requirements.txt
+2024-08-23 20:27:56,429 INFO    SenderThread:13176 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_202540-om09pls8/files/config.yaml config.yaml
+2024-08-23 20:27:56,431 INFO    SenderThread:13176 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_202540-om09pls8/files/wandb-metadata.json wandb-metadata.json
+2024-08-23 20:27:56,431 INFO    SenderThread:13176 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json wandb-summary.json
+2024-08-23 20:27:56,433 INFO    SenderThread:13176 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_202540-om09pls8/files/output.log output.log
+2024-08-23 20:27:56,433 INFO    SenderThread:13176 [sender.py:transition_state():617] send defer: 10
+2024-08-23 20:27:56,434 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 20:27:56,435 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:56,435 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 10
+2024-08-23 20:27:56,436 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:56,436 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 10
+2024-08-23 20:27:56,436 INFO    SenderThread:13176 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-23 20:27:56,841 INFO    wandb-upload_0:13176 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_202540-om09pls8/files/requirements.txt
+2024-08-23 20:27:56,894 INFO    wandb-upload_2:13176 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json
+2024-08-23 20:27:56,896 INFO    wandb-upload_3:13176 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_202540-om09pls8/files/output.log
+2024-08-23 20:27:56,902 INFO    wandb-upload_1:13176 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_202540-om09pls8/files/config.yaml
+2024-08-23 20:27:57,013 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 20:27:57,014 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 20:27:57,103 INFO    Thread-11 (_thread_body):13176 [sender.py:transition_state():617] send defer: 11
+2024-08-23 20:27:57,103 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:57,103 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 11
+2024-08-23 20:27:57,103 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:57,103 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 11
+2024-08-23 20:27:57,103 INFO    SenderThread:13176 [file_pusher.py:join():178] waiting for file pusher
+2024-08-23 20:27:57,103 INFO    SenderThread:13176 [sender.py:transition_state():617] send defer: 12
+2024-08-23 20:27:57,103 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:57,103 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 12
+2024-08-23 20:27:57,104 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:57,104 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 12
+2024-08-23 20:27:57,104 INFO    SenderThread:13176 [file_stream.py:finish():595] file stream finish called
+2024-08-23 20:27:57,389 INFO    SenderThread:13176 [file_stream.py:finish():599] file stream finish is done
+2024-08-23 20:27:57,389 INFO    SenderThread:13176 [sender.py:transition_state():617] send defer: 13
+2024-08-23 20:27:57,389 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:57,389 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 13
+2024-08-23 20:27:57,389 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:57,389 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 13
+2024-08-23 20:27:57,389 INFO    SenderThread:13176 [sender.py:transition_state():617] send defer: 14
+2024-08-23 20:27:57,390 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 20:27:57,390 DEBUG   SenderThread:13176 [sender.py:send():382] send: final
+2024-08-23 20:27:57,390 INFO    HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 14
+2024-08-23 20:27:57,390 DEBUG   SenderThread:13176 [sender.py:send():382] send: footer
+2024-08-23 20:27:57,390 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: defer
+2024-08-23 20:27:57,390 INFO    SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 14
+2024-08-23 20:27:57,391 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 20:27:57,391 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 20:27:57,391 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 20:27:57,391 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: server_info
+2024-08-23 20:27:57,392 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 20:27:57,392 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: get_summary
+2024-08-23 20:27:57,392 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: server_info
+2024-08-23 20:27:57,393 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: sampled_history
+2024-08-23 20:27:57,395 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 20:27:57,395 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: job_info
+2024-08-23 20:27:57,563 DEBUG   SenderThread:13176 [sender.py:send_request():409] send_request: job_info
+2024-08-23 20:27:57,563 INFO    MainThread:13176 [wandb_run.py:_footer_history_summary_info():3866] rendering history
+2024-08-23 20:27:57,564 INFO    MainThread:13176 [wandb_run.py:_footer_history_summary_info():3898] rendering summary
+2024-08-23 20:27:57,564 INFO    MainThread:13176 [wandb_run.py:_footer_sync_info():3825] logging synced files
+2024-08-23 20:27:57,564 DEBUG   HandlerThread:13176 [handler.py:handle_request():146] handle_request: shutdown
+2024-08-23 20:27:57,564 INFO    HandlerThread:13176 [handler.py:finish():869] shutting down handler
+2024-08-23 20:27:58,396 INFO    WriterThread:13176 [datastore.py:close():296] close: /project/wandb/run-20240823_202540-om09pls8/run-om09pls8.wandb
+2024-08-23 20:27:58,563 INFO    SenderThread:13176 [sender.py:finish():1572] shutting down sender
+2024-08-23 20:27:58,563 INFO    SenderThread:13176 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-23 20:27:58,563 INFO    SenderThread:13176 [file_pusher.py:join():178] waiting for file pusher

wandb/run-20240823_202540-om09pls8/logs/debug.log

@@ -0,0 +1,30 @@
+2024-08-23 20:25:40,742 INFO    MainThread:12857 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
+2024-08-23 20:25:40,743 INFO    MainThread:12857 [wandb_setup.py:_flush():76] Configure stats pid to 12857
+2024-08-23 20:25:40,743 INFO    MainThread:12857 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings
+2024-08-23 20:25:40,743 INFO    MainThread:12857 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings
+2024-08-23 20:25:40,743 INFO    MainThread:12857 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'}
+2024-08-23 20:25:40,743 INFO    MainThread:12857 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-08-23 20:25:40,743 INFO    MainThread:12857 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'}
+2024-08-23 20:25:40,743 INFO    MainThread:12857 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240823_202540-om09pls8/logs/debug.log
+2024-08-23 20:25:40,743 INFO    MainThread:12857 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240823_202540-om09pls8/logs/debug-internal.log
+2024-08-23 20:25:40,743 INFO    MainThread:12857 [wandb_init.py:init():566] calling init triggers
+2024-08-23 20:25:40,743 INFO    MainThread:12857 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
+config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'test_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'Qwen2-0.5b-0.2_train_2024-08-23-20:25:00', 'wandb_project': 'llm_tutorial-0.2', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'save': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 3, 'save_interval': 500, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 16000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 16000, 'train_samples': None, 'global_batch_size': 612, 'micro_batch_size': 17, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/Qwen2-0.5b-0.2', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 10, 'rank': 0, 'world_size': 4, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 9}
+2024-08-23 20:25:40,743 INFO    MainThread:12857 [wandb_init.py:init():616] starting backend
+2024-08-23 20:25:40,743 INFO    MainThread:12857 [wandb_init.py:init():620] setting up manager
+2024-08-23 20:25:40,748 INFO    MainThread:12857 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-08-23 20:25:40,750 INFO    MainThread:12857 [wandb_init.py:init():628] backend started and connected
+2024-08-23 20:25:40,754 INFO    MainThread:12857 [wandb_init.py:init():720] updated telemetry
+2024-08-23 20:25:40,864 INFO    MainThread:12857 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-08-23 20:25:41,341 INFO    MainThread:12857 [wandb_run.py:_on_init():2262] communicating current version
+2024-08-23 20:25:41,364 INFO    MainThread:12857 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-08-23 20:25:41,364 INFO    MainThread:12857 [wandb_init.py:init():804] starting run threads in backend
+2024-08-23 20:25:41,512 INFO    MainThread:12857 [wandb_run.py:_console_start():2241] atexit reg
+2024-08-23 20:25:41,512 INFO    MainThread:12857 [wandb_run.py:_redirect():2096] redirect: wrap_raw
+2024-08-23 20:25:41,512 INFO    MainThread:12857 [wandb_run.py:_redirect():2161] Wrapping output streams.
+2024-08-23 20:25:41,513 INFO    MainThread:12857 [wandb_run.py:_redirect():2186] Redirects installed.
+2024-08-23 20:25:41,514 INFO    MainThread:12857 [wandb_init.py:init():847] run started, returning control to user process
+2024-08-23 20:26:02,594 INFO    MainThread:12857 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 1024, 'num_attention_heads': 14, 'num_hidden_layers': 24}
+2024-08-23 20:26:02,595 INFO    MainThread:12857 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 4}
+2024-08-23 20:27:58,565 WARNING MsgRouterThr:12857 [router.py:message_loop():77] message_loop has been closed

wandb/run-20240831_192346-5vo4p2k7/files/config.yaml

@@ -0,0 +1,313 @@
+wandb_version: 1
+
+sharding_strategy:
+  desc: null
+  value: SHARD_GRAD_OP
+checkpoint_type:
+  desc: null
+  value: LOCAL_STATE_DICT
+fsdp_activation_checkpointing:
+  desc: null
+  value: true
+fsdp_cpu_offload:
+  desc: null
+  value: false
+low_cpu_fsdp:
+  desc: null
+  value: false
+no_meta_device:
+  desc: null
+  value: false
+data_path:
+  desc: null
+  value: null
+split:
+  desc: null
+  value: 969, 30, 1
+train_data_path:
+  desc: null
+  value: null
+valid_data_path:
+  desc: null
+  value: null
+test_data_path:
+  desc: null
+  value: null
+data_cache_path:
+  desc: null
+  value: null
+vocab_size:
+  desc: null
+  value: null
+vocab_file:
+  desc: null
+  value: null
+merge_file:
+  desc: null
+  value: null
+seq_length:
+  desc: null
+  value: 2048
+num_workers:
+  desc: null
+  value: 4
+tokenizer_type:
+  desc: null
+  value: HFPreTrainedTokenizer
+tokenizer_model:
+  desc: null
+  value: /share/pretrained_lm/Qwen/Qwen2-0.5B
+reset_position_ids:
+  desc: null
+  value: false
+reset_attention_mask:
+  desc: null
+  value: false
+eod_mask_loss:
+  desc: null
+  value: false
+retro_return_doc_ids:
+  desc: null
+  value: false
+short_seq_prob:
+  desc: null
+  value: 0.1
+vocab_extra_ids:
+  desc: null
+  value: 0
+seed:
+  desc: null
+  value: 1234
+use_mpi:
+  desc: null
+  value: false
+wandb_entity:
+  desc: null
+  value: iwakawa-koichi-q5-tohoku-nlp6723
+wandb_name:
+  desc: null
+  value: yans-baseline-qwen2-0.5B-3.5e-5-ichikara_train_2024-08-31-19:23:33
+wandb_project:
+  desc: null
+  value: yans_experiment
+quantization:
+  desc: null
+  value: false
+use_freeze_layers:
+  desc: null
+  value: false
+freeze_layers:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+mixed_precision:
+  desc: null
+  value: true
+param_dtype:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara
+save:
+  desc: null
+  value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara
+base_model:
+  desc: null
+  value: /share/pretrained_lm/Qwen/Qwen2-0.5B
+use_better_transformer:
+  desc: null
+  value: false
+grad_clip_norm:
+  desc: null
+  value: 1.0
+eval_interval:
+  desc: null
+  value: 100
+save_interval:
+  desc: null
+  value: 100
+eval_iters:
+  desc: null
+  value: 10
+optimizer:
+  desc: null
+  value: anyprecision
+lr:
+  desc: null
+  value: 2.0e-05
+lr_decay_style:
+  desc: null
+  value: cosine
+lr_decay_iters:
+  desc: null
+  value: 1000
+lr_warmup_iters:
+  desc: null
+  value: 25
+min_lr:
+  desc: null
+  value: 1.0e-06
+train_iters:
+  desc: null
+  value: 1000
+train_samples:
+  desc: null
+  value: null
+global_batch_size:
+  desc: null
+  value: 16
+micro_batch_size:
+  desc: null
+  value: 4
+make_vocab_size_divisible_by:
+  desc: null
+  value: 128
+sliding_window_size:
+  desc: null
+  value: 4096
+skip_batch:
+  desc: null
+  value: null
+no_save_optimizer_state:
+  desc: null
+  value: false
+continual_pretraining:
+  desc: null
+  value: false
+instruction_tuning:
+  desc: null
+  value: false
+direct_preference_optimization:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.1
+hidden_dropout:
+  desc: null
+  value: 0.1
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.99
+adam_eps:
+  desc: null
+  value: 1.0e-06
+hf_transformer_model_dir:
+  desc: null
+  value: /share/pretrained_lm/Qwen/Qwen2-0.5B
+instruction_train_data_path:
+  desc: null
+  value: /work/datasets/bin/ichikara/train/data
+instruction_valid_data_path:
+  desc: null
+  value: /work/datasets/bin/ichikara/valid/data
+epoch:
+  desc: null
+  value: null
+instruction_dataset_size:
+  desc: null
+  value: null
+save_sampler_state:
+  desc: null
+  value: true
+label_smoothing:
+  desc: null
+  value: 0.0
+save_n_checkpoints:
+  desc: null
+  value: 10
+hf_repo_id:
+  desc: null
+  value: koichi12/yans-baseline-qwen2-0.5B-3.5e-5-ichikara
+create_public_hf_repo:
+  desc: null
+  value: false
+upload_all_checkpoints_to_hf:
+  desc: null
+  value: false
+hf_upload_retry_limit:
+  desc: null
+  value: 2
+exit_duration_in_mins:
+  desc: null
+  value: null
+source_key:
+  desc: null
+  value: source
+target_key:
+  desc: null
+  value: target
+attn_implementation:
+  desc: null
+  value: flash_attention_2
+efficient_instruction_tuning:
+  desc: null
+  value: true
+remove_padding_masking:
+  desc: null
+  value: true
+save_start_iter:
+  desc: null
+  value: null
+valid_micro_batch_size:
+  desc: null
+  value: 1
+rank:
+  desc: null
+  value: 0
+world_size:
+  desc: null
+  value: 1
+padded_vocab_size:
+  desc: null
+  value: 151680
+gradient_accumulation_steps:
+  desc: null
+  value: 4
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.43.3
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1725099826.21653
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      - 105
+      2:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      - 105
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.43.3
+      8:
+      - 5
+      13: linux-x86_64

wandb/run-20240831_192346-5vo4p2k7/files/output.log

@@ -0,0 +1,15 @@
+Created Hugging Face repository with ID koichi12/yans-baseline-qwen2-0.5B-3.5e-5-ichikara.
+Clearing GPU cache for all ranks
+--> Running with torch torch_distributed debug set to detail
+File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara/latest_iteration.txt
+File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara/latest_iteration.txt
+Traceback (most recent call last):
+  File "/project/examples/finetuning.py", line 13, in <module>
+    main()
+  File "/project/src/llama_recipes/finetuning.py", line 103, in main
+    model = get_model(
+  File "/project/src/llama_recipes/get_models.py", line 106, in get_model
+    assert sliding_window == 131072
+AssertionError

wandb/run-20240831_192346-5vo4p2k7/files/requirements.txt

@@ -0,0 +1,375 @@
+absl-py==2.1.0
+accelerate==0.23.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+apex==0.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+astroid==3.2.4
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+beautifulsoup4==4.12.3
+bert-score==0.3.13
+bleach==6.1.0
+blis==0.7.11
+build==1.2.1
+cachecontrol==0.14.0
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+cleo==2.1.0
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+cmake==3.28.1
+colorama==0.4.6
+comm==0.2.1
+confection==0.1.4
+contourpy==1.2.0
+cramjam==2.8.3
+crashtest==0.4.1
+cryptography==43.0.0
+cubinlinker==0.3.0+2.g405ac64
+cuda-python==12.3.0rc4+9.gdb8c48a.dirty
+cudf==23.12.0
+cugraph-dgl==23.12.0
+cugraph-service-client==23.12.0
+cugraph-service-server==23.12.0
+cugraph==23.12.0
+cuml==23.12.0
+cupy-cuda12x==12.3.0
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.8
+dask-cuda==23.12.0
+dask-cudf==23.12.0
+dask==2023.11.0
+dataclasses-json==0.6.7
+dataproperty==1.0.1
+datasets==2.20.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.8
+distlib==0.3.8
+distributed==2023.11.0
+distro==1.9.0
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+dulwich==0.21.7
+einops==0.7.0
+emoji==2.12.1
+entmax==1.3
+evaluate==0.4.2
+exceptiongroup==1.2.0
+execnet==2.0.2
+executing==2.0.1
+expecttest==0.1.3
+fastjsonschema==2.19.1
+fastparquet==2023.10.1
+fastrlock==0.8.2
+filelock==3.13.1
+flash-attn==2.4.2
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2023.12.2
+fugashi==1.3.2
+fuzzywuzzy==0.18.0
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.27.0
+graphsurgeon==0.4.6
+greenlet==3.0.3
+grpcio==1.60.1
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.24.5
+hydra-core==1.3.2
+hypothesis==5.35.1
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+installer==0.7.0
+intel-openmp==2021.4.0
+ipadic==1.0.0
+ipykernel==6.29.2
+ipython-genutils==0.2.0
+ipython==8.21.0
+isort==5.13.2
+jaraco.classes==3.4.0
+jedi==0.19.1
+jeepney==0.8.0
+jinja2==3.1.3
+jiter==0.5.0
+joblib==1.3.2
+json5==0.9.14
+jsonargparse==3.13.1
+jsonlines==4.0.0
+jsonnet==0.19.1
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema-specifications==2023.12.1
+jsonschema==4.21.1
+jupyter-client==8.6.0
+jupyter-core==5.7.1
+jupyter-tensorboard==0.2.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==1.2.0
+jupyterlab==2.3.2
+jupytext==1.16.1
+keyring==24.3.1
+kiwisolver==1.4.5
+langchain-community==0.2.12
+langchain-core==0.2.31
+langchain-huggingface==0.0.2
+langchain-openai==0.1.21
+langchain-text-splitters==0.2.2
+langchain==0.2.13
+langcodes==3.3.0
+langsmith==0.1.99
+lazy-loader==0.3
+levenshtein==0.25.1
+librosa==0.10.1
+lightning-utilities==0.11.6
+llm-jp-eval==1.4.0
+llvmlite==0.40.1
+lm-eval==0.3.0
+locket==1.0.0
+logzero==1.7.0
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown==3.5.2
+markupsafe==2.1.4
+marshmallow==3.21.3
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mbstrdecoder==1.1.3
+mccabe==0.7.0
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mecab-python3==1.0.6
+mistune==3.0.2
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mkl==2021.1.1
+mock==5.1.0
+mojimoji==0.0.13
+more-itertools==9.1.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+multiprocess==0.70.16
+murmurhash==1.0.10
+mypy-extensions==1.0.0
+nbclient==0.9.0
+nbconvert==7.16.0
+nbformat==5.9.2
+neologdn==0.5.3
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.4.10
+numba==0.57.1+1.g1ff679645
+numexpr==2.10.1
+numpy==1.24.4
+nvfuser==0.1.4a0+d0bb811
+nvidia-dali-cuda120==1.34.0
+nvidia-pyindex==1.0.9
+nvtx==0.2.5
+oauthlib==3.2.2
+omegaconf==2.3.0
+onnx==1.15.0rc2
+openai==1.40.6
+opencv==4.7.0
+optree==0.10.0
+orjson==3.10.7
+packaging==23.2
+pandas==2.2.2
+pandocfilters==1.5.1
+parso==0.8.3
+partd==1.4.1
+pathvalidate==3.2.0
+peft==0.5.0
+pexpect==4.9.0
+pillow==10.2.0
+pip==24.0
+pkginfo==1.11.1
+plac==1.4.3
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+poetry-core==1.9.0
+poetry-plugin-export==1.8.0
+poetry==1.8.3
+polygraphy==0.49.4
+pooch==1.8.0
+portalocker==2.10.1
+preshed==3.0.9
+prettytable==3.9.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.24.4
+psutil==5.9.4
+ptxcompiler==0.8.1+2.g0d406d6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow-hotfix==0.6
+pyarrow==15.0.2
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pybind11-global==2.11.1
+pybind11==2.11.1
+pycocotools==2.0+nv0.8.0
+pycountry==24.6.1
+pycparser==2.21
+pydantic-core==2.16.2
+pydantic==2.6.1
+pygments==2.17.2
+pylibcugraph==23.12.0
+pylibcugraphops==23.12.0
+pylibraft==23.12.0
+pylint==3.2.6
+pynvml==11.4.1
+pyparsing==3.1.1
+pyproject-hooks==1.1.0
+pytablewriter==1.2.0
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==13.0
+pytest-shard==0.1.2
+pytest-xdist==3.5.0
+pytest==8.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-hostlist==1.23.0
+python-levenshtein==0.25.1
+pytorch-lightning==2.4.0
+pytorch-quantization==2.1.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.2
+raft-dask==23.12.0
+rapidfuzz==3.9.6
+rapids-dask-dependency==23.12.1
+referencing==0.33.0
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests-toolbelt==1.0.0
+requests==2.32.3
+rhoknp==1.7.0
+rich==13.7.0
+rmm==23.12.0
+rouge-score==0.1.2
+rpds-py==0.17.1
+rsa==4.9
+sacrebleu==2.4.2
+safetensors==0.4.3
+scikit-learn==1.5.1
+scipy==1.12.0
+secretstorage==3.3.3
+send2trash==1.8.2
+sentence-transformers==3.0.1
+sentencepiece==0.1.99
+sentry-sdk==2.12.0
+setproctitle==1.3.3
+setuptools==68.2.2
+shellingham==1.5.4
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sniffio==1.3.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sphinx-glpi-theme==0.6
+sqlalchemy==2.0.32
+sqlitedict==2.1.0
+srsly==2.4.8
+stack-data==0.6.3
+sumeval==0.2.2
+sympy==1.12
+tabledata==1.3.3
+tabulate==0.9.0
+tbb==2021.11.0
+tblib==3.0.0
+tcolorpy==0.1.6
+tenacity==8.5.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.9.0
+tensorrt==8.6.3
+terminado==0.18.0
+termplotlib==0.3.9
+text-generation==0.7.0
+thinc==8.2.3
+threadpoolctl==3.2.0
+thriftpy2==0.4.17
+tiktoken==0.7.0
+tinycss2==1.2.1
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+tomlkit==0.13.2
+toolz==0.12.1
+torch-tensorrt==2.3.0a0
+torch==2.3.0a0+ebedce2
+torchdata==0.7.1a0
+torchmetrics==0.10.3
+torchtext==0.17.0a0
+torchvision==0.18.0a0
+tornado==6.4
+tqdm-multiprocess==0.0.11
+tqdm==4.66.5
+traitlets==5.9.0
+transformer-engine==1.3.0+5b90b7f
+transformers==4.43.3
+treelite-runtime==3.9.1
+treelite==3.9.1
+triton==2.2.0+e28a256
+trove-classifiers==2024.7.2
+typepy==1.3.2
+typer==0.9.0
+types-dataclasses==0.6.6
+typing-extensions==4.12.2
+typing-inspect==0.9.0
+tzdata==2024.1
+ucx-py==0.35.0
+uff==0.6.9
+ujson==5.8.0
+unbabel-comet==2.2.2
+unidic-lite==1.0.8
+urllib3==1.26.18
+virtualenv==20.26.3
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.3.4
+webencodings==0.5.1
+werkzeug==3.0.1
+wheel==0.42.0
+word2number==1.1
+xdoctest==1.0.2
+xgboost==1.7.6
+xmltodict==0.13.0
+xxhash==3.4.1
+yarl==1.9.4
+zict==3.0.0
+zipp==3.17.0
+zstandard==0.23.0

wandb/run-20240831_192346-5vo4p2k7/files/wandb-metadata.json

@@ -0,0 +1,221 @@
+{
+    "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-08-31T10:23:46.789150",
+    "startedAt": "2024-08-31T10:23:46.204215",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--seq-length",
+        "2048",
+        "--micro-batch-size",
+        "4",
+        "--valid_micro_batch_size",
+        "1",
+        "--global-batch-size",
+        "16",
+        "--train-iters",
+        "1000",
+        "--tokenizer-type",
+        "HFPreTrainedTokenizer",
+        "--tokenizer-model",
+        "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+        "--instruction-train-data-path",
+        "/work/datasets/bin/ichikara/train/data",
+        "--instruction-valid-data-path",
+        "/work/datasets/bin/ichikara/valid/data",
+        "--efficient-instruction-tuning",
+        "--remove-padding-masking",
+        "--source-key",
+        "source",
+        "--target-key",
+        "target",
+        "--lr",
+        "2e-5",
+        "--min-lr",
+        "1e-6",
+        "--lr-decay-style",
+        "cosine",
+        "--lr-warmup-iters",
+        "25",
+        "--lr-decay-iters",
+        "1000",
+        "--weight-decay",
+        "0.1",
+        "--grad-clip-norm",
+        "1.0",
+        "--optimizer",
+        "anyprecision",
+        "--adam-beta1",
+        "0.9",
+        "--adam-beta2",
+        "0.99",
+        "--adam-eps",
+        "1e-6",
+        "--save-interval",
+        "100",
+        "--eval-interval",
+        "100",
+        "--eval-iters",
+        "10",
+        "--bf16",
+        "--mixed-precision",
+        "--base-model",
+        "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+        "--hf-transformer-model-dir",
+        "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+        "--save",
+        "/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara",
+        "--load",
+        "/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara",
+        "--fsdp-activation-checkpointing",
+        "--sharding-strategy",
+        "SHARD_GRAD_OP",
+        "--checkpoint-type",
+        "LOCAL_STATE_DICT",
+        "--save-sampler-state",
+        "--save-n-checkpoints",
+        "10",
+        "--hf-upload-retry-limit",
+        "2",
+        "--hf-repo-id",
+        "koichi12/yans-baseline-qwen2-0.5B-3.5e-5-ichikara",
+        "--num-workers",
+        "4",
+        "--wandb-entity",
+        "iwakawa-koichi-q5-tohoku-nlp6723",
+        "--wandb-project",
+        "yans_experiment",
+        "--wandb-name",
+        "yans-baseline-qwen2-0.5B-3.5e-5-ichikara_train_2024-08-31-19:23:33"
+    ],
+    "state": "running",
+    "program": "/project/examples/finetuning.py",
+    "codePathLocal": "examples/finetuning.py",
+    "codePath": "examples/finetuning.py",
+    "git": {
+        "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git",
+        "commit": "3b2976faebe2228c39adb20194a29b785a37defe"
+    },
+    "email": null,
+    "root": "/project",
+    "host": "gpu-koiwa-00",
+    "username": "koiwa",
+    "executable": "/usr/bin/python",
+    "cpu_count": 18,
+    "cpu_count_logical": 18,
+    "cpu_freq": {
+        "current": 2400.025999999999,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.026,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 0.0625,
+            "used": 1.1444091796875e-05
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 56.48781967163086
+    }
+}

wandb/run-20240831_192346-5vo4p2k7/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 3}}