Add files using upload-large-folder tool

wandb/run-20240804_021032-cd2cg2ui/files/config.yaml

@@ -0,0 +1,314 @@
+wandb_version: 1
+
+sharding_strategy:
+  desc: null
+  value: FULL_SHARD
+checkpoint_type:
+  desc: null
+  value: LOCAL_STATE_DICT
+fsdp_activation_checkpointing:
+  desc: null
+  value: true
+fsdp_cpu_offload:
+  desc: null
+  value: false
+low_cpu_fsdp:
+  desc: null
+  value: false
+no_meta_device:
+  desc: null
+  value: false
+data_path:
+  desc: null
+  value: null
+split:
+  desc: null
+  value: 969, 30, 1
+train_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+valid_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+test_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+data_cache_path:
+  desc: null
+  value: null
+vocab_size:
+  desc: null
+  value: null
+vocab_file:
+  desc: null
+  value: null
+merge_file:
+  desc: null
+  value: null
+seq_length:
+  desc: null
+  value: 1024
+num_workers:
+  desc: null
+  value: 2
+tokenizer_type:
+  desc: null
+  value: Llama2Tokenizer
+tokenizer_model:
+  desc: null
+  value: /share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3
+reset_position_ids:
+  desc: null
+  value: false
+reset_attention_mask:
+  desc: null
+  value: false
+eod_mask_loss:
+  desc: null
+  value: false
+retro_return_doc_ids:
+  desc: null
+  value: false
+short_seq_prob:
+  desc: null
+  value: 0.1
+vocab_extra_ids:
+  desc: null
+  value: 0
+seed:
+  desc: null
+  value: 1234
+use_mpi:
+  desc: null
+  value: false
+wandb_entity:
+  desc: null
+  value: iwakawa-koichi-q5-tohoku-nlp6723
+wandb_name:
+  desc: null
+  value: tiny-mistral-sample_train_2024-08-04-02:10:14
+wandb_project:
+  desc: null
+  value: llm_tutorial
+quantization:
+  desc: null
+  value: false
+use_freeze_layers:
+  desc: null
+  value: false
+freeze_layers:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+mixed_precision:
+  desc: null
+  value: true
+param_dtype:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: /work/llm_recipes/models/tiny-mistral-sample
+save:
+  desc: null
+  value: /work/llm_recipes/models/tiny-mistral-sample
+base_model:
+  desc: null
+  value: /share/pretrained_lm/custom/tiny-mistral
+use_better_transformer:
+  desc: null
+  value: false
+grad_clip_norm:
+  desc: null
+  value: 1.0
+eval_interval:
+  desc: null
+  value: 200
+save_interval:
+  desc: null
+  value: 200
+eval_iters:
+  desc: null
+  value: 10
+optimizer:
+  desc: null
+  value: adam
+lr:
+  desc: null
+  value: 2.0e-05
+lr_decay_style:
+  desc: null
+  value: cosine
+lr_decay_iters:
+  desc: null
+  value: 20000
+lr_warmup_iters:
+  desc: null
+  value: 500
+min_lr:
+  desc: null
+  value: 1.0e-06
+train_iters:
+  desc: null
+  value: 20000
+train_samples:
+  desc: null
+  value: null
+global_batch_size:
+  desc: null
+  value: 320
+micro_batch_size:
+  desc: null
+  value: 8
+make_vocab_size_divisible_by:
+  desc: null
+  value: 128
+sliding_window_size:
+  desc: null
+  value: 8192
+skip_batch:
+  desc: null
+  value: null
+no_save_optimizer_state:
+  desc: null
+  value: false
+continual_pretraining:
+  desc: null
+  value: false
+instruction_tuning:
+  desc: null
+  value: false
+direct_preference_optimization:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.1
+hidden_dropout:
+  desc: null
+  value: 0.1
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_eps:
+  desc: null
+  value: 1.0e-06
+hf_transformer_model_dir:
+  desc: null
+  value: null
+instruction_train_data_path:
+  desc: null
+  value: null
+instruction_valid_data_path:
+  desc: null
+  value: null
+epoch:
+  desc: null
+  value: null
+instruction_dataset_size:
+  desc: null
+  value: null
+save_sampler_state:
+  desc: null
+  value: false
+label_smoothing:
+  desc: null
+  value: 0.0
+save_n_checkpoints:
+  desc: null
+  value: 10
+hf_repo_id:
+  desc: null
+  value: koichi12/tiny-mistral-sample
+create_public_hf_repo:
+  desc: null
+  value: false
+upload_all_checkpoints_to_hf:
+  desc: null
+  value: false
+hf_upload_retry_limit:
+  desc: null
+  value: 2
+exit_duration_in_mins:
+  desc: null
+  value: null
+source_key:
+  desc: null
+  value: null
+target_key:
+  desc: null
+  value: null
+attn_implementation:
+  desc: null
+  value: flash_attention_2
+efficient_instruction_tuning:
+  desc: null
+  value: false
+remove_padding_masking:
+  desc: null
+  value: false
+save_start_iter:
+  desc: null
+  value: null
+rank:
+  desc: null
+  value: 0
+world_size:
+  desc: null
+  value: 1
+padded_vocab_size:
+  desc: null
+  value: 32768
+gradient_accumulation_steps:
+  desc: null
+  value: 40
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.43.3
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1722705032.417279
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      2:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.43.3
+      8:
+      - 5
+      13: linux-x86_64

wandb/run-20240804_021032-cd2cg2ui/files/requirements.txt

@@ -0,0 +1,271 @@
+absl-py==2.1.0
+accelerate==0.33.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+apex==0.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+beautifulsoup4==4.12.3
+bleach==6.1.0
+blis==0.7.11
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+cmake==3.28.1
+colorama==0.4.6
+comm==0.2.1
+confection==0.1.4
+contourpy==1.2.0
+cubinlinker==0.3.0+2.g405ac64
+cuda-python==12.3.0rc4+9.gdb8c48a.dirty
+cudf==23.12.0
+cugraph-dgl==23.12.0
+cugraph-service-client==23.12.0
+cugraph-service-server==23.12.0
+cugraph==23.12.0
+cuml==23.12.0
+cupy-cuda12x==12.3.0
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.8
+dask-cuda==23.12.0
+dask-cudf==23.12.0
+dask==2023.11.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+distributed==2023.11.0
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+einops==0.7.0
+exceptiongroup==1.2.0
+execnet==2.0.2
+executing==2.0.1
+expecttest==0.1.3
+fastjsonschema==2.19.1
+fastrlock==0.8.2
+filelock==3.13.1
+flash-attn==2.4.2
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2023.12.2
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.27.0
+graphsurgeon==0.4.6
+grpcio==1.60.1
+huggingface-hub==0.24.5
+hypothesis==5.35.1
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+intel-openmp==2021.4.0
+ipadic==1.0.0
+ipykernel==6.29.2
+ipython-genutils==0.2.0
+ipython==8.21.0
+jedi==0.19.1
+jinja2==3.1.3
+joblib==1.3.2
+json5==0.9.14
+jsonnet==0.19.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.21.1
+jupyter-client==8.6.0
+jupyter-core==5.7.1
+jupyter-tensorboard==0.2.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==1.2.0
+jupyterlab==2.3.2
+jupytext==1.16.1
+kiwisolver==1.4.5
+langcodes==3.3.0
+lazy-loader==0.3
+librosa==0.10.1
+llvmlite==0.40.1
+locket==1.0.0
+logzero==1.7.0
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown==3.5.2
+markupsafe==2.1.4
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mecab-python3==1.0.6
+mistune==3.0.2
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mkl==2021.1.1
+mock==5.1.0
+more-itertools==9.1.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+murmurhash==1.0.10
+nbclient==0.9.0
+nbconvert==7.16.0
+nbformat==5.9.2
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.4.10
+numba==0.57.1+1.g1ff679645
+numpy==1.24.4
+nvfuser==0.1.4a0+d0bb811
+nvidia-dali-cuda120==1.34.0
+nvidia-pyindex==1.0.9
+nvtx==0.2.5
+oauthlib==3.2.2
+onnx==1.15.0rc2
+opencv==4.7.0
+optree==0.10.0
+packaging==23.2
+pandas==1.5.3
+pandocfilters==1.5.1
+parso==0.8.3
+partd==1.4.1
+peft==0.11.1
+pexpect==4.9.0
+pillow==10.2.0
+pip==24.0
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+polygraphy==0.49.4
+pooch==1.8.0
+portalocker==2.10.1
+preshed==3.0.9
+prettytable==3.9.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.24.4
+psutil==5.9.4
+ptxcompiler==0.8.1+2.g0d406d6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==14.0.1.dev0+gba5374836.d20240125
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pybind11-global==2.11.1
+pybind11==2.11.1
+pycocotools==2.0+nv0.8.0
+pycparser==2.21
+pydantic-core==2.16.2
+pydantic==2.6.1
+pygments==2.17.2
+pylibcugraph==23.12.0
+pylibcugraphops==23.12.0
+pylibraft==23.12.0
+pynvml==11.4.1
+pyparsing==3.1.1
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==13.0
+pytest-shard==0.1.2
+pytest-xdist==3.5.0
+pytest==8.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-hostlist==1.23.0
+pytorch-quantization==2.1.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.2
+raft-dask==23.12.0
+rapids-dask-dependency==23.12.1
+referencing==0.33.0
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests==2.31.0
+rich==13.7.0
+rmm==23.12.0
+rpds-py==0.17.1
+rsa==4.9
+sacrebleu==2.4.0
+safetensors==0.4.3
+scikit-learn==1.2.0
+scipy==1.12.0
+send2trash==1.8.2
+sentencepiece==0.1.99
+sentry-sdk==2.12.0
+setproctitle==1.3.3
+setuptools==68.2.2
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sphinx-glpi-theme==0.6
+srsly==2.4.8
+stack-data==0.6.3
+sympy==1.12
+tabulate==0.9.0
+tbb==2021.11.0
+tblib==3.0.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.9.0
+tensorrt==8.6.3
+terminado==0.18.0
+termplotlib==0.3.9
+thinc==8.2.3
+threadpoolctl==3.2.0
+thriftpy2==0.4.17
+tinycss2==1.2.1
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.1
+torch-tensorrt==2.3.0a0
+torch==2.3.0a0+ebedce2
+torchdata==0.7.1a0
+torchtext==0.17.0a0
+torchvision==0.18.0a0
+tornado==6.4
+tqdm==4.66.1
+traitlets==5.9.0
+transformer-engine==1.3.0+5b90b7f
+transformers==4.43.3
+treelite-runtime==3.9.1
+treelite==3.9.1
+triton==2.2.0+e28a256
+typer==0.9.0
+types-dataclasses==0.6.6
+typing-extensions==4.9.0
+ucx-py==0.35.0
+uff==0.6.9
+ujson==5.8.0
+urllib3==1.26.18
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.3.4
+webencodings==0.5.1
+werkzeug==3.0.1
+wheel==0.42.0
+xdoctest==1.0.2
+xgboost==1.7.6
+yarl==1.9.4
+zict==3.0.0
+zipp==3.17.0

wandb/run-20240804_021032-cd2cg2ui/logs/debug-internal.log

@@ -0,0 +1,185 @@
+2024-08-04 02:10:32,669 INFO    StreamThr :11309 [internal.py:wandb_internal():86] W&B internal server running at pid: 11309, started at: 2024-08-04 02:10:32.417731
+2024-08-04 02:10:32,670 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: status
+2024-08-04 02:10:32,672 INFO    WriterThread:11309 [datastore.py:open_for_write():87] open: /project/wandb/run-20240804_021032-cd2cg2ui/run-cd2cg2ui.wandb
+2024-08-04 02:10:32,673 DEBUG   SenderThread:11309 [sender.py:send():382] send: header
+2024-08-04 02:10:32,883 DEBUG   SenderThread:11309 [sender.py:send():382] send: run
+2024-08-04 02:10:33,348 INFO    SenderThread:11309 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240804_021032-cd2cg2ui/files
+2024-08-04 02:10:33,348 INFO    SenderThread:11309 [sender.py:_start_run_threads():1136] run started: cd2cg2ui with start time 1722705032.417279
+2024-08-04 02:10:33,353 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: check_version
+2024-08-04 02:10:33,354 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: check_version
+2024-08-04 02:10:33,438 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: run_start
+2024-08-04 02:10:33,444 DEBUG   HandlerThread:11309 [system_info.py:__init__():27] System info init
+2024-08-04 02:10:33,444 DEBUG   HandlerThread:11309 [system_info.py:__init__():42] System info init done
+2024-08-04 02:10:33,444 INFO    HandlerThread:11309 [system_monitor.py:start():194] Starting system monitor
+2024-08-04 02:10:33,444 INFO    SystemMonitor:11309 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-08-04 02:10:33,445 INFO    HandlerThread:11309 [system_monitor.py:probe():214] Collecting system info
+2024-08-04 02:10:33,445 INFO    SystemMonitor:11309 [interfaces.py:start():190] Started cpu monitoring
+2024-08-04 02:10:33,445 INFO    SystemMonitor:11309 [interfaces.py:start():190] Started disk monitoring
+2024-08-04 02:10:33,447 INFO    SystemMonitor:11309 [interfaces.py:start():190] Started gpu monitoring
+2024-08-04 02:10:33,447 INFO    SystemMonitor:11309 [interfaces.py:start():190] Started memory monitoring
+2024-08-04 02:10:33,448 INFO    SystemMonitor:11309 [interfaces.py:start():190] Started network monitoring
+2024-08-04 02:10:33,458 DEBUG   HandlerThread:11309 [system_info.py:probe():151] Probing system
+2024-08-04 02:10:33,460 DEBUG   HandlerThread:11309 [system_info.py:_probe_git():136] Probing git
+2024-08-04 02:10:33,471 DEBUG   HandlerThread:11309 [system_info.py:_probe_git():144] Probing git done
+2024-08-04 02:10:33,471 DEBUG   HandlerThread:11309 [system_info.py:probe():199] Probing system done
+2024-08-04 02:10:33,471 DEBUG   HandlerThread:11309 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-03T17:10:33.458421', 'startedAt': '2024-08-03T17:10:32.395506', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--sliding-window-size', '8192', '--micro-batch-size', '8', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/custom/tiny-mistral', '--save', '/work/llm_recipes/models/tiny-mistral-sample', '--load', '/work/llm_recipes/models/tiny-mistral-sample', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-mistral-sample', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-mistral-sample_train_2024-08-04-02:10:14'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.034, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}}
+2024-08-04 02:10:33,471 INFO    HandlerThread:11309 [system_monitor.py:probe():224] Finished collecting system info
+2024-08-04 02:10:33,471 INFO    HandlerThread:11309 [system_monitor.py:probe():227] Publishing system info
+2024-08-04 02:10:33,473 INFO    HandlerThread:11309 [system_monitor.py:probe():229] Finished publishing system info
+2024-08-04 02:10:33,478 DEBUG   SenderThread:11309 [sender.py:send():382] send: files
+2024-08-04 02:10:33,479 INFO    SenderThread:11309 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now
+2024-08-04 02:10:33,488 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: python_packages
+2024-08-04 02:10:33,488 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-04 02:10:33,488 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: python_packages
+2024-08-04 02:10:33,489 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-04 02:10:33,490 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: stop_status
+2024-08-04 02:10:33,776 DEBUG   SenderThread:11309 [sender.py:send():382] send: telemetry
+2024-08-04 02:10:34,131 INFO    wandb-upload_0:11309 [upload_job.py:push():131] Uploaded file /tmp/tmpcp1trk59wandb/1uhn5dog-wandb-metadata.json
+2024-08-04 02:10:34,327 DEBUG   SenderThread:11309 [sender.py:send():382] send: exit
+2024-08-04 02:10:34,327 INFO    SenderThread:11309 [sender.py:send_exit():589] handling exit code: 1
+2024-08-04 02:10:34,327 INFO    SenderThread:11309 [sender.py:send_exit():591] handling runtime: 0
+2024-08-04 02:10:34,328 INFO    SenderThread:11309 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-04 02:10:34,329 INFO    SenderThread:11309 [sender.py:send_exit():597] send defer
+2024-08-04 02:10:34,329 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:34,329 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 0
+2024-08-04 02:10:34,329 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:34,329 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 0
+2024-08-04 02:10:34,329 INFO    SenderThread:11309 [sender.py:transition_state():617] send defer: 1
+2024-08-04 02:10:34,329 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:34,329 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 1
+2024-08-04 02:10:34,330 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:34,330 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 1
+2024-08-04 02:10:34,330 INFO    SenderThread:11309 [sender.py:transition_state():617] send defer: 2
+2024-08-04 02:10:34,330 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:34,330 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 2
+2024-08-04 02:10:34,330 INFO    HandlerThread:11309 [system_monitor.py:finish():203] Stopping system monitor
+2024-08-04 02:10:34,330 DEBUG   SystemMonitor:11309 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-08-04 02:10:34,330 INFO    HandlerThread:11309 [interfaces.py:finish():202] Joined cpu monitor
+2024-08-04 02:10:34,330 DEBUG   SystemMonitor:11309 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-08-04 02:10:34,331 INFO    HandlerThread:11309 [interfaces.py:finish():202] Joined disk monitor
+2024-08-04 02:10:34,331 DEBUG   SystemMonitor:11309 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-08-04 02:10:34,350 INFO    Thread-12 :11309 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_021032-cd2cg2ui/files/requirements.txt
+2024-08-04 02:10:34,350 INFO    Thread-12 :11309 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_021032-cd2cg2ui/files/output.log
+2024-08-04 02:10:34,350 INFO    Thread-12 :11309 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_021032-cd2cg2ui/files/wandb-summary.json
+2024-08-04 02:10:34,350 INFO    Thread-12 :11309 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_021032-cd2cg2ui/files/wandb-metadata.json
+2024-08-04 02:10:34,365 INFO    HandlerThread:11309 [interfaces.py:finish():202] Joined gpu monitor
+2024-08-04 02:10:34,366 INFO    HandlerThread:11309 [interfaces.py:finish():202] Joined memory monitor
+2024-08-04 02:10:34,366 INFO    HandlerThread:11309 [interfaces.py:finish():202] Joined network monitor
+2024-08-04 02:10:34,366 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:34,366 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 2
+2024-08-04 02:10:34,366 INFO    SenderThread:11309 [sender.py:transition_state():617] send defer: 3
+2024-08-04 02:10:34,366 DEBUG   SenderThread:11309 [sender.py:send():382] send: stats
+2024-08-04 02:10:34,366 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:34,367 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 3
+2024-08-04 02:10:34,367 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:34,367 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 3
+2024-08-04 02:10:34,367 INFO    SenderThread:11309 [sender.py:transition_state():617] send defer: 4
+2024-08-04 02:10:34,367 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:34,367 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 4
+2024-08-04 02:10:34,367 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:34,367 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 4
+2024-08-04 02:10:34,367 INFO    SenderThread:11309 [sender.py:transition_state():617] send defer: 5
+2024-08-04 02:10:34,367 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:34,367 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 5
+2024-08-04 02:10:34,368 DEBUG   SenderThread:11309 [sender.py:send():382] send: summary
+2024-08-04 02:10:34,368 INFO    SenderThread:11309 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-04 02:10:34,369 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:34,369 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 5
+2024-08-04 02:10:34,369 INFO    SenderThread:11309 [sender.py:transition_state():617] send defer: 6
+2024-08-04 02:10:34,369 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:34,369 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 6
+2024-08-04 02:10:34,369 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:34,369 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 6
+2024-08-04 02:10:34,372 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 02:10:34,573 INFO    SenderThread:11309 [sender.py:transition_state():617] send defer: 7
+2024-08-04 02:10:34,573 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:34,573 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 7
+2024-08-04 02:10:34,573 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:34,573 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 7
+2024-08-04 02:10:35,327 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-04 02:10:35,350 INFO    Thread-12 :11309 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_021032-cd2cg2ui/files/config.yaml
+2024-08-04 02:10:36,318 INFO    SenderThread:11309 [sender.py:transition_state():617] send defer: 8
+2024-08-04 02:10:36,318 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: poll_exit
+2024-08-04 02:10:36,318 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:36,318 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 8
+2024-08-04 02:10:36,318 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:36,318 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 8
+2024-08-04 02:10:36,319 INFO    SenderThread:11309 [job_builder.py:build():296] Attempting to build job artifact
+2024-08-04 02:10:36,319 INFO    SenderThread:11309 [job_builder.py:_get_source_type():426] is repo sourced job
+2024-08-04 02:10:36,328 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-04 02:10:36,333 INFO    SenderThread:11309 [job_builder.py:build():402] adding wandb-job metadata file
+2024-08-04 02:10:36,342 INFO    SenderThread:11309 [sender.py:transition_state():617] send defer: 9
+2024-08-04 02:10:36,342 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: poll_exit
+2024-08-04 02:10:36,342 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:36,342 DEBUG   SenderThread:11309 [sender.py:send():382] send: artifact
+2024-08-04 02:10:36,342 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 9
+2024-08-04 02:10:36,351 INFO    Thread-12 :11309 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_021032-cd2cg2ui/files/output.log
+2024-08-04 02:10:37,328 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-04 02:10:37,552 INFO    wandb-upload_1:11309 [upload_job.py:push():86] Skipped uploading /singularity_home/.local/share/wandb/artifacts/staging/tmpteaibpd9
+2024-08-04 02:10:37,910 INFO    wandb-upload_0:11309 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmp9hfu5wh3
+2024-08-04 02:10:39,216 INFO    SenderThread:11309 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA5MzUzODM4NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA5MTk4ODAyMA==', 'versionIndex': 2}}}
+2024-08-04 02:10:39,216 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:39,216 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 9
+2024-08-04 02:10:39,216 INFO    SenderThread:11309 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-08-04 02:10:39,352 INFO    SenderThread:11309 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240804_021032-cd2cg2ui/files
+2024-08-04 02:10:39,352 INFO    SenderThread:11309 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021032-cd2cg2ui/files/requirements.txt requirements.txt
+2024-08-04 02:10:39,352 INFO    SenderThread:11309 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021032-cd2cg2ui/files/config.yaml config.yaml
+2024-08-04 02:10:39,354 INFO    SenderThread:11309 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021032-cd2cg2ui/files/wandb-metadata.json wandb-metadata.json
+2024-08-04 02:10:39,354 INFO    SenderThread:11309 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021032-cd2cg2ui/files/wandb-summary.json wandb-summary.json
+2024-08-04 02:10:39,355 INFO    SenderThread:11309 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_021032-cd2cg2ui/files/output.log output.log
+2024-08-04 02:10:39,357 INFO    SenderThread:11309 [sender.py:transition_state():617] send defer: 10
+2024-08-04 02:10:39,357 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: poll_exit
+2024-08-04 02:10:39,358 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:39,358 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 10
+2024-08-04 02:10:39,359 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:39,359 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 10
+2024-08-04 02:10:39,359 INFO    SenderThread:11309 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-04 02:10:39,788 INFO    wandb-upload_0:11309 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_021032-cd2cg2ui/files/config.yaml
+2024-08-04 02:10:39,856 INFO    wandb-upload_1:11309 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_021032-cd2cg2ui/files/requirements.txt
+2024-08-04 02:10:39,931 INFO    wandb-upload_3:11309 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_021032-cd2cg2ui/files/output.log
+2024-08-04 02:10:39,937 INFO    wandb-upload_2:11309 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_021032-cd2cg2ui/files/wandb-summary.json
+2024-08-04 02:10:40,137 INFO    Thread-11 (_thread_body):11309 [sender.py:transition_state():617] send defer: 11
+2024-08-04 02:10:40,137 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:40,137 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 11
+2024-08-04 02:10:40,138 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:40,138 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 11
+2024-08-04 02:10:40,138 INFO    SenderThread:11309 [file_pusher.py:join():178] waiting for file pusher
+2024-08-04 02:10:40,138 INFO    SenderThread:11309 [sender.py:transition_state():617] send defer: 12
+2024-08-04 02:10:40,138 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:40,138 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 12
+2024-08-04 02:10:40,138 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:40,138 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 12
+2024-08-04 02:10:40,138 INFO    SenderThread:11309 [file_stream.py:finish():595] file stream finish called
+2024-08-04 02:10:40,324 INFO    SenderThread:11309 [file_stream.py:finish():599] file stream finish is done
+2024-08-04 02:10:40,324 INFO    SenderThread:11309 [sender.py:transition_state():617] send defer: 13
+2024-08-04 02:10:40,324 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:40,324 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 13
+2024-08-04 02:10:40,324 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:40,324 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 13
+2024-08-04 02:10:40,324 INFO    SenderThread:11309 [sender.py:transition_state():617] send defer: 14
+2024-08-04 02:10:40,325 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 02:10:40,325 DEBUG   SenderThread:11309 [sender.py:send():382] send: final
+2024-08-04 02:10:40,325 INFO    HandlerThread:11309 [handler.py:handle_request_defer():172] handle defer: 14
+2024-08-04 02:10:40,325 DEBUG   SenderThread:11309 [sender.py:send():382] send: footer
+2024-08-04 02:10:40,325 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: defer
+2024-08-04 02:10:40,325 INFO    SenderThread:11309 [sender.py:send_request_defer():613] handle sender defer: 14
+2024-08-04 02:10:40,325 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-04 02:10:40,326 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-04 02:10:40,326 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: poll_exit
+2024-08-04 02:10:40,326 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: server_info
+2024-08-04 02:10:40,326 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: poll_exit
+2024-08-04 02:10:40,326 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: server_info
+2024-08-04 02:10:40,328 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: get_summary
+2024-08-04 02:10:40,328 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: sampled_history
+2024-08-04 02:10:40,328 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-04 02:10:40,329 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: job_info
+2024-08-04 02:10:40,492 DEBUG   SenderThread:11309 [sender.py:send_request():409] send_request: job_info
+2024-08-04 02:10:40,492 INFO    MainThread:11309 [wandb_run.py:_footer_history_summary_info():3866] rendering history
+2024-08-04 02:10:40,492 INFO    MainThread:11309 [wandb_run.py:_footer_history_summary_info():3898] rendering summary
+2024-08-04 02:10:40,492 INFO    MainThread:11309 [wandb_run.py:_footer_sync_info():3825] logging synced files
+2024-08-04 02:10:40,492 DEBUG   HandlerThread:11309 [handler.py:handle_request():146] handle_request: shutdown
+2024-08-04 02:10:40,492 INFO    HandlerThread:11309 [handler.py:finish():869] shutting down handler
+2024-08-04 02:10:41,329 INFO    WriterThread:11309 [datastore.py:close():296] close: /project/wandb/run-20240804_021032-cd2cg2ui/run-cd2cg2ui.wandb
+2024-08-04 02:10:41,492 INFO    SenderThread:11309 [sender.py:finish():1572] shutting down sender
+2024-08-04 02:10:41,492 INFO    SenderThread:11309 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-04 02:10:41,492 INFO    SenderThread:11309 [file_pusher.py:join():178] waiting for file pusher

wandb/run-20240804_021032-cd2cg2ui/logs/debug.log

@@ -0,0 +1,28 @@
+2024-08-04 02:10:32,410 INFO    MainThread:11238 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
+2024-08-04 02:10:32,411 INFO    MainThread:11238 [wandb_setup.py:_flush():76] Configure stats pid to 11238
+2024-08-04 02:10:32,411 INFO    MainThread:11238 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings
+2024-08-04 02:10:32,411 INFO    MainThread:11238 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings
+2024-08-04 02:10:32,411 INFO    MainThread:11238 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tuny llama sample'}
+2024-08-04 02:10:32,411 INFO    MainThread:11238 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-08-04 02:10:32,411 INFO    MainThread:11238 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'}
+2024-08-04 02:10:32,411 INFO    MainThread:11238 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240804_021032-cd2cg2ui/logs/debug.log
+2024-08-04 02:10:32,411 INFO    MainThread:11238 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240804_021032-cd2cg2ui/logs/debug-internal.log
+2024-08-04 02:10:32,411 INFO    MainThread:11238 [wandb_init.py:init():566] calling init triggers
+2024-08-04 02:10:32,411 INFO    MainThread:11238 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
+config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-mistral-sample_train_2024-08-04-02:10:14', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-mistral-sample', 'save': '/work/llm_recipes/models/tiny-mistral-sample', 'base_model': '/share/pretrained_lm/custom/tiny-mistral', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 8, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 8192, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-mistral-sample', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32768, 'gradient_accumulation_steps': 40}
+2024-08-04 02:10:32,411 INFO    MainThread:11238 [wandb_init.py:init():616] starting backend
+2024-08-04 02:10:32,411 INFO    MainThread:11238 [wandb_init.py:init():620] setting up manager
+2024-08-04 02:10:32,416 INFO    MainThread:11238 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-08-04 02:10:32,416 INFO    MainThread:11238 [wandb_init.py:init():628] backend started and connected
+2024-08-04 02:10:32,422 INFO    MainThread:11238 [wandb_init.py:init():720] updated telemetry
+2024-08-04 02:10:32,879 INFO    MainThread:11238 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-08-04 02:10:33,353 INFO    MainThread:11238 [wandb_run.py:_on_init():2262] communicating current version
+2024-08-04 02:10:33,431 INFO    MainThread:11238 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-08-04 02:10:33,431 INFO    MainThread:11238 [wandb_init.py:init():804] starting run threads in backend
+2024-08-04 02:10:33,487 INFO    MainThread:11238 [wandb_run.py:_console_start():2241] atexit reg
+2024-08-04 02:10:33,487 INFO    MainThread:11238 [wandb_run.py:_redirect():2096] redirect: wrap_raw
+2024-08-04 02:10:33,487 INFO    MainThread:11238 [wandb_run.py:_redirect():2161] Wrapping output streams.
+2024-08-04 02:10:33,488 INFO    MainThread:11238 [wandb_run.py:_redirect():2186] Redirects installed.
+2024-08-04 02:10:33,489 INFO    MainThread:11238 [wandb_init.py:init():847] run started, returning control to user process
+2024-08-04 02:10:41,493 WARNING MsgRouterThr:11238 [router.py:message_loop():77] message_loop has been closed

wandb/run-20240804_035140-nyllt780/files/config.yaml

@@ -0,0 +1,335 @@
+wandb_version: 1
+
+sharding_strategy:
+  desc: null
+  value: FULL_SHARD
+checkpoint_type:
+  desc: null
+  value: LOCAL_STATE_DICT
+fsdp_activation_checkpointing:
+  desc: null
+  value: true
+fsdp_cpu_offload:
+  desc: null
+  value: false
+low_cpu_fsdp:
+  desc: null
+  value: false
+no_meta_device:
+  desc: null
+  value: false
+data_path:
+  desc: null
+  value: null
+split:
+  desc: null
+  value: 969, 30, 1
+train_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+valid_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+test_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+data_cache_path:
+  desc: null
+  value: null
+vocab_size:
+  desc: null
+  value: null
+vocab_file:
+  desc: null
+  value: null
+merge_file:
+  desc: null
+  value: null
+seq_length:
+  desc: null
+  value: 512
+num_workers:
+  desc: null
+  value: 2
+tokenizer_type:
+  desc: null
+  value: Llama2Tokenizer
+tokenizer_model:
+  desc: null
+  value: /share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model
+reset_position_ids:
+  desc: null
+  value: false
+reset_attention_mask:
+  desc: null
+  value: false
+eod_mask_loss:
+  desc: null
+  value: false
+retro_return_doc_ids:
+  desc: null
+  value: false
+short_seq_prob:
+  desc: null
+  value: 0.1
+vocab_extra_ids:
+  desc: null
+  value: 0
+seed:
+  desc: null
+  value: 1234
+use_mpi:
+  desc: null
+  value: false
+wandb_entity:
+  desc: null
+  value: iwakawa-koichi-q5-tohoku-nlp6723
+wandb_name:
+  desc: null
+  value: tiny-llama-sample_train_2024-08-04-03:51:30
+wandb_project:
+  desc: null
+  value: llm_tutorial
+quantization:
+  desc: null
+  value: false
+use_freeze_layers:
+  desc: null
+  value: false
+freeze_layers:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+mixed_precision:
+  desc: null
+  value: true
+param_dtype:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: /work/llm_recipes/models/tiny-llama-sample
+save:
+  desc: null
+  value: /work/llm_recipes/models/tiny-llama-sample
+base_model:
+  desc: null
+  value: /share/pretrained_lm/meta-llama/TinyLlama_v1.1
+use_better_transformer:
+  desc: null
+  value: false
+grad_clip_norm:
+  desc: null
+  value: 1.0
+eval_interval:
+  desc: null
+  value: 200
+save_interval:
+  desc: null
+  value: 200
+eval_iters:
+  desc: null
+  value: 10
+optimizer:
+  desc: null
+  value: adam
+lr:
+  desc: null
+  value: 2.0e-05
+lr_decay_style:
+  desc: null
+  value: cosine
+lr_decay_iters:
+  desc: null
+  value: 20000
+lr_warmup_iters:
+  desc: null
+  value: 500
+min_lr:
+  desc: null
+  value: 1.0e-06
+train_iters:
+  desc: null
+  value: 20000
+train_samples:
+  desc: null
+  value: null
+global_batch_size:
+  desc: null
+  value: 320
+micro_batch_size:
+  desc: null
+  value: 8
+make_vocab_size_divisible_by:
+  desc: null
+  value: 128
+sliding_window_size:
+  desc: null
+  value: 4096
+skip_batch:
+  desc: null
+  value: null
+no_save_optimizer_state:
+  desc: null
+  value: false
+continual_pretraining:
+  desc: null
+  value: false
+instruction_tuning:
+  desc: null
+  value: false
+direct_preference_optimization:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.1
+hidden_dropout:
+  desc: null
+  value: 0.1
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_eps:
+  desc: null
+  value: 1.0e-06
+hf_transformer_model_dir:
+  desc: null
+  value: null
+instruction_train_data_path:
+  desc: null
+  value: null
+instruction_valid_data_path:
+  desc: null
+  value: null
+epoch:
+  desc: null
+  value: null
+instruction_dataset_size:
+  desc: null
+  value: null
+save_sampler_state:
+  desc: null
+  value: false
+label_smoothing:
+  desc: null
+  value: 0.0
+save_n_checkpoints:
+  desc: null
+  value: 10
+hf_repo_id:
+  desc: null
+  value: koichi12/tiny-llama-sample
+create_public_hf_repo:
+  desc: null
+  value: false
+upload_all_checkpoints_to_hf:
+  desc: null
+  value: false
+hf_upload_retry_limit:
+  desc: null
+  value: 2
+exit_duration_in_mins:
+  desc: null
+  value: null
+source_key:
+  desc: null
+  value: null
+target_key:
+  desc: null
+  value: null
+attn_implementation:
+  desc: null
+  value: flash_attention_2
+efficient_instruction_tuning:
+  desc: null
+  value: false
+remove_padding_masking:
+  desc: null
+  value: false
+save_start_iter:
+  desc: null
+  value: null
+rank:
+  desc: null
+  value: 0
+world_size:
+  desc: null
+  value: 1
+padded_vocab_size:
+  desc: null
+  value: 32000
+gradient_accumulation_steps:
+  desc: null
+  value: 40
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.43.3
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1722711100.510646
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      2:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.43.3
+      8:
+      - 5
+      13: linux-x86_64
+activation_function:
+  desc: null
+  value: silu
+hidden_size:
+  desc: null
+  value: 2048
+model_type:
+  desc: null
+  value: llama
+max_position_embeddings:
+  desc: null
+  value: 2048
+num_attention_heads:
+  desc: null
+  value: 32
+num_hidden_layers:
+  desc: null
+  value: 22
+model_architecture:
+  desc: null
+  value: LlamaForCausalLM

wandb/run-20240804_035140-nyllt780/files/output.log

@@ -0,0 +1,130 @@
+Created Hugging Face repository with ID koichi12/tiny-llama-sample.
+Clearing GPU cache for all ranks
+--> Running with torch torch_distributed debug set to detail
+File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt
+File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt
+You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)`
+Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaModel is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)`
+File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt
+No checkpoint found in /work/llm_recipes/models/tiny-llama-sample, skipping model loading
+--> Model /share/pretrained_lm/meta-llama/TinyLlama_v1.1
+--> /share/pretrained_lm/meta-llama/TinyLlama_v1.1 has 1100.048384 Million params
+/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1.
+  warnings.warn(
+Let split = None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+BFloat16 enabled for mixed precision - using bfSixteen policy
+--> applying fsdp activation checkpointing...
+ > datasets target sizes (minimum size):
+    train:      6400000
+    validation: 323200
+    test:       3200
+> building train, validation, and test datasets for GPT ...
+> finished creating GPT datasets ...
+File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt
+No checkpoint found in /work/llm_recipes/models/tiny-llama-sample, skipping optimizer loading
+File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt
+model info: FullyShardedDataParallel(
+  (_fsdp_wrapped_module): LlamaForCausalLM(
+    (model): LlamaModel(
+      (embed_tokens): Embedding(32000, 2048)
+      (layers): ModuleList(
+        (0-21): 22 x FullyShardedDataParallel(
+          (_fsdp_wrapped_module): CheckpointWrapper(
+            (_checkpoint_wrapped_module): LlamaDecoderLayer(
+              (self_attn): LlamaFlashAttention2(
+                (q_proj): Linear(in_features=2048, out_features=2048, bias=False)
+                (k_proj): Linear(in_features=2048, out_features=256, bias=False)
+                (v_proj): Linear(in_features=2048, out_features=256, bias=False)
+                (o_proj): Linear(in_features=2048, out_features=2048, bias=False)
+                (rotary_emb): LlamaRotaryEmbedding()
+              )
+              (mlp): LlamaMLP(
+                (gate_proj): Linear(in_features=2048, out_features=5632, bias=False)
+                (up_proj): Linear(in_features=2048, out_features=5632, bias=False)
+                (down_proj): Linear(in_features=5632, out_features=2048, bias=False)
+                (act_fn): SiLU()
+              )
+              (input_layernorm): LlamaRMSNorm()
+              (post_attention_layernorm): LlamaRMSNorm()
+            )
+          )
+        )
+      )
+      (norm): LlamaRMSNorm()
+      (rotary_emb): LlamaRotaryEmbedding()
+    )
+    (lm_head): Linear(in_features=2048, out_features=32000, bias=False)
+  )
+)
+model config: LlamaConfig {
+  "_name_or_path": "/share/pretrained_lm/meta-llama/TinyLlama_v1.1",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5632,
+  "label_smoothing": 0.0,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 22,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.43.3",
+  "use_cache": false,
+  "vocab_size": 32000
+}
+Traceback (most recent call last):
+  File "/project/examples/finetuning.py", line 13, in <module>
+    main()
+  File "/project/src/llama_recipes/finetuning.py", line 281, in main
+    train(
+  File "/project/src/llama_recipes/utils/train_utils.py", line 110, in train
+    loss: torch.Tensor = model(**batch).loss
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 849, in forward
+    output = self._fsdp_wrapped_module(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/project/lib/transformers/src/transformers/models/llama/modeling_llama.py", line 1141, in forward
+    outputs = self.model(
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/project/lib/transformers/src/transformers/models/llama/modeling_llama.py", line 908, in forward
+    cache_position = torch.arange(
+RuntimeError: CUDA error: device-side assert triggered
+CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
+For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
+Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

wandb/run-20240804_035140-nyllt780/files/requirements.txt

@@ -0,0 +1,271 @@
+absl-py==2.1.0
+accelerate==0.33.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+apex==0.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+beautifulsoup4==4.12.3
+bleach==6.1.0
+blis==0.7.11
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+cmake==3.28.1
+colorama==0.4.6
+comm==0.2.1
+confection==0.1.4
+contourpy==1.2.0
+cubinlinker==0.3.0+2.g405ac64
+cuda-python==12.3.0rc4+9.gdb8c48a.dirty
+cudf==23.12.0
+cugraph-dgl==23.12.0
+cugraph-service-client==23.12.0
+cugraph-service-server==23.12.0
+cugraph==23.12.0
+cuml==23.12.0
+cupy-cuda12x==12.3.0
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.8
+dask-cuda==23.12.0
+dask-cudf==23.12.0
+dask==2023.11.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+distributed==2023.11.0
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+einops==0.7.0
+exceptiongroup==1.2.0
+execnet==2.0.2
+executing==2.0.1
+expecttest==0.1.3
+fastjsonschema==2.19.1
+fastrlock==0.8.2
+filelock==3.13.1
+flash-attn==2.4.2
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2023.12.2
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.27.0
+graphsurgeon==0.4.6
+grpcio==1.60.1
+huggingface-hub==0.24.5
+hypothesis==5.35.1
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+intel-openmp==2021.4.0
+ipadic==1.0.0
+ipykernel==6.29.2
+ipython-genutils==0.2.0
+ipython==8.21.0
+jedi==0.19.1
+jinja2==3.1.3
+joblib==1.3.2
+json5==0.9.14
+jsonnet==0.19.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.21.1
+jupyter-client==8.6.0
+jupyter-core==5.7.1
+jupyter-tensorboard==0.2.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==1.2.0
+jupyterlab==2.3.2
+jupytext==1.16.1
+kiwisolver==1.4.5
+langcodes==3.3.0
+lazy-loader==0.3
+librosa==0.10.1
+llvmlite==0.40.1
+locket==1.0.0
+logzero==1.7.0
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown==3.5.2
+markupsafe==2.1.4
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mecab-python3==1.0.6
+mistune==3.0.2
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mkl==2021.1.1
+mock==5.1.0
+more-itertools==9.1.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+murmurhash==1.0.10
+nbclient==0.9.0
+nbconvert==7.16.0
+nbformat==5.9.2
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.4.10
+numba==0.57.1+1.g1ff679645
+numpy==1.24.4
+nvfuser==0.1.4a0+d0bb811
+nvidia-dali-cuda120==1.34.0
+nvidia-pyindex==1.0.9
+nvtx==0.2.5
+oauthlib==3.2.2
+onnx==1.15.0rc2
+opencv==4.7.0
+optree==0.10.0
+packaging==23.2
+pandas==1.5.3
+pandocfilters==1.5.1
+parso==0.8.3
+partd==1.4.1
+peft==0.11.1
+pexpect==4.9.0
+pillow==10.2.0
+pip==24.0
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+polygraphy==0.49.4
+pooch==1.8.0
+portalocker==2.10.1
+preshed==3.0.9
+prettytable==3.9.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.24.4
+psutil==5.9.4
+ptxcompiler==0.8.1+2.g0d406d6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==14.0.1.dev0+gba5374836.d20240125
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pybind11-global==2.11.1
+pybind11==2.11.1
+pycocotools==2.0+nv0.8.0
+pycparser==2.21
+pydantic-core==2.16.2
+pydantic==2.6.1
+pygments==2.17.2
+pylibcugraph==23.12.0
+pylibcugraphops==23.12.0
+pylibraft==23.12.0
+pynvml==11.4.1
+pyparsing==3.1.1
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==13.0
+pytest-shard==0.1.2
+pytest-xdist==3.5.0
+pytest==8.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-hostlist==1.23.0
+pytorch-quantization==2.1.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.2
+raft-dask==23.12.0
+rapids-dask-dependency==23.12.1
+referencing==0.33.0
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests==2.31.0
+rich==13.7.0
+rmm==23.12.0
+rpds-py==0.17.1
+rsa==4.9
+sacrebleu==2.4.0
+safetensors==0.4.3
+scikit-learn==1.2.0
+scipy==1.12.0
+send2trash==1.8.2
+sentencepiece==0.1.99
+sentry-sdk==2.12.0
+setproctitle==1.3.3
+setuptools==68.2.2
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sphinx-glpi-theme==0.6
+srsly==2.4.8
+stack-data==0.6.3
+sympy==1.12
+tabulate==0.9.0
+tbb==2021.11.0
+tblib==3.0.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.9.0
+tensorrt==8.6.3
+terminado==0.18.0
+termplotlib==0.3.9
+thinc==8.2.3
+threadpoolctl==3.2.0
+thriftpy2==0.4.17
+tinycss2==1.2.1
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.1
+torch-tensorrt==2.3.0a0
+torch==2.3.0a0+ebedce2
+torchdata==0.7.1a0
+torchtext==0.17.0a0
+torchvision==0.18.0a0
+tornado==6.4
+tqdm==4.66.1
+traitlets==5.9.0
+transformer-engine==1.3.0+5b90b7f
+transformers==4.43.3
+treelite-runtime==3.9.1
+treelite==3.9.1
+triton==2.2.0+e28a256
+typer==0.9.0
+types-dataclasses==0.6.6
+typing-extensions==4.9.0
+ucx-py==0.35.0
+uff==0.6.9
+ujson==5.8.0
+urllib3==1.26.18
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.3.4
+webencodings==0.5.1
+werkzeug==3.0.1
+wheel==0.42.0
+xdoctest==1.0.2
+xgboost==1.7.6
+yarl==1.9.4
+zict==3.0.0
+zipp==3.17.0

wandb/run-20240804_035140-nyllt780/files/wandb-metadata.json

@@ -0,0 +1,215 @@
+{
+    "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-08-03T18:51:41.236802",
+    "startedAt": "2024-08-03T18:51:40.498160",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--seq-length",
+        "512",
+        "--sliding-window-size",
+        "4096",
+        "--micro-batch-size",
+        "8",
+        "--global-batch-size",
+        "320",
+        "--train-iters",
+        "20000",
+        "--tokenizer-type",
+        "Llama2Tokenizer",
+        "--tokenizer-model",
+        "/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model",
+        "--train-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--valid-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--test-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--lr",
+        "2e-5",
+        "--min-lr",
+        "1e-6",
+        "--lr-decay-style",
+        "cosine",
+        "--lr-warmup-iters",
+        "500",
+        "--lr-decay-iters",
+        "20000",
+        "--weight-decay",
+        "0.1",
+        "--grad-clip-norm",
+        "1.0",
+        "--optimizer",
+        "adam",
+        "--adam-beta1",
+        "0.9",
+        "--adam-beta2",
+        "0.95",
+        "--adam-eps",
+        "1e-6",
+        "--save-interval",
+        "200",
+        "--eval-interval",
+        "200",
+        "--eval-iters",
+        "10",
+        "--bf16",
+        "--mixed-precision",
+        "--base-model",
+        "/share/pretrained_lm/meta-llama/TinyLlama_v1.1",
+        "--save",
+        "/work/llm_recipes/models/tiny-llama-sample",
+        "--load",
+        "/work/llm_recipes/models/tiny-llama-sample",
+        "--fsdp-activation-checkpointing",
+        "--sharding-strategy",
+        "FULL_SHARD",
+        "--checkpoint-type",
+        "LOCAL_STATE_DICT",
+        "--save-n-checkpoints",
+        "10",
+        "--hf-upload-retry-limit",
+        "2",
+        "--hf-repo-id",
+        "koichi12/tiny-llama-sample",
+        "--wandb-entity",
+        "iwakawa-koichi-q5-tohoku-nlp6723",
+        "--wandb-project",
+        "llm_tutorial",
+        "--wandb-name",
+        "tiny-llama-sample_train_2024-08-04-03:51:30"
+    ],
+    "state": "running",
+    "program": "/project/examples/finetuning.py",
+    "codePathLocal": "examples/finetuning.py",
+    "codePath": "examples/finetuning.py",
+    "git": {
+        "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git",
+        "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea"
+    },
+    "email": null,
+    "root": "/project",
+    "host": "gpu-koiwa-00",
+    "username": "koiwa",
+    "executable": "/usr/bin/python",
+    "cpu_count": 18,
+    "cpu_count_logical": 18,
+    "cpu_freq": {
+        "current": 2400.034,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.034,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 0.0625,
+            "used": 1.1444091796875e-05
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 56.48782730102539
+    }
+}

wandb/run-20240804_035140-nyllt780/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 63}}

wandb/run-20240804_035140-nyllt780/logs/debug-internal.log

@@ -0,0 +1,215 @@
+2024-08-04 03:51:40,525 INFO    StreamThr :12425 [internal.py:wandb_internal():86] W&B internal server running at pid: 12425, started at: 2024-08-04 03:51:40.511089
+2024-08-04 03:51:40,526 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: status
+2024-08-04 03:51:40,527 INFO    WriterThread:12425 [datastore.py:open_for_write():87] open: /project/wandb/run-20240804_035140-nyllt780/run-nyllt780.wandb
+2024-08-04 03:51:40,541 DEBUG   SenderThread:12425 [sender.py:send():382] send: header
+2024-08-04 03:51:40,658 DEBUG   SenderThread:12425 [sender.py:send():382] send: run
+2024-08-04 03:51:41,127 INFO    SenderThread:12425 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240804_035140-nyllt780/files
+2024-08-04 03:51:41,127 INFO    SenderThread:12425 [sender.py:_start_run_threads():1136] run started: nyllt780 with start time 1722711100.510646
+2024-08-04 03:51:41,132 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: check_version
+2024-08-04 03:51:41,133 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: check_version
+2024-08-04 03:51:41,218 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: run_start
+2024-08-04 03:51:41,224 DEBUG   HandlerThread:12425 [system_info.py:__init__():27] System info init
+2024-08-04 03:51:41,224 DEBUG   HandlerThread:12425 [system_info.py:__init__():42] System info init done
+2024-08-04 03:51:41,224 INFO    HandlerThread:12425 [system_monitor.py:start():194] Starting system monitor
+2024-08-04 03:51:41,224 INFO    SystemMonitor:12425 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-08-04 03:51:41,224 INFO    HandlerThread:12425 [system_monitor.py:probe():214] Collecting system info
+2024-08-04 03:51:41,225 INFO    SystemMonitor:12425 [interfaces.py:start():190] Started cpu monitoring
+2024-08-04 03:51:41,225 INFO    SystemMonitor:12425 [interfaces.py:start():190] Started disk monitoring
+2024-08-04 03:51:41,226 INFO    SystemMonitor:12425 [interfaces.py:start():190] Started gpu monitoring
+2024-08-04 03:51:41,227 INFO    SystemMonitor:12425 [interfaces.py:start():190] Started memory monitoring
+2024-08-04 03:51:41,228 INFO    SystemMonitor:12425 [interfaces.py:start():190] Started network monitoring
+2024-08-04 03:51:41,236 DEBUG   HandlerThread:12425 [system_info.py:probe():151] Probing system
+2024-08-04 03:51:41,238 DEBUG   HandlerThread:12425 [system_info.py:_probe_git():136] Probing git
+2024-08-04 03:51:41,249 DEBUG   HandlerThread:12425 [system_info.py:_probe_git():144] Probing git done
+2024-08-04 03:51:41,249 DEBUG   HandlerThread:12425 [system_info.py:probe():199] Probing system done
+2024-08-04 03:51:41,250 DEBUG   HandlerThread:12425 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-03T18:51:41.236802', 'startedAt': '2024-08-03T18:51:40.498160', 'docker': None, 'cuda': None, 'args': ('--seq-length', '512', '--sliding-window-size', '4096', '--micro-batch-size', '8', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/meta-llama/TinyLlama_v1.1', '--save', '/work/llm_recipes/models/tiny-llama-sample', '--load', '/work/llm_recipes/models/tiny-llama-sample', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-llama-sample', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-llama-sample_train_2024-08-04-03:51:30'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.034, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}}
+2024-08-04 03:51:41,250 INFO    HandlerThread:12425 [system_monitor.py:probe():224] Finished collecting system info
+2024-08-04 03:51:41,250 INFO    HandlerThread:12425 [system_monitor.py:probe():227] Publishing system info
+2024-08-04 03:51:41,302 INFO    HandlerThread:12425 [system_monitor.py:probe():229] Finished publishing system info
+2024-08-04 03:51:41,308 DEBUG   SenderThread:12425 [sender.py:send():382] send: files
+2024-08-04 03:51:41,308 INFO    SenderThread:12425 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now
+2024-08-04 03:51:41,317 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: python_packages
+2024-08-04 03:51:41,317 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-04 03:51:41,317 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-04 03:51:41,317 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: python_packages
+2024-08-04 03:51:41,335 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: stop_status
+2024-08-04 03:51:41,618 DEBUG   SenderThread:12425 [sender.py:send():382] send: telemetry
+2024-08-04 03:51:41,985 INFO    wandb-upload_0:12425 [upload_job.py:push():131] Uploaded file /tmp/tmpxkt1klm7wandb/bxmu94ae-wandb-metadata.json
+2024-08-04 03:51:42,129 INFO    Thread-12 :12425 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_035140-nyllt780/files/output.log
+2024-08-04 03:51:42,129 INFO    Thread-12 :12425 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_035140-nyllt780/files/requirements.txt
+2024-08-04 03:51:42,129 INFO    Thread-12 :12425 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_035140-nyllt780/files/wandb-metadata.json
+2024-08-04 03:51:44,129 INFO    Thread-12 :12425 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_035140-nyllt780/files/output.log
+2024-08-04 03:51:45,608 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 03:51:48,132 INFO    Thread-12 :12425 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_035140-nyllt780/files/output.log
+2024-08-04 03:51:50,610 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 03:51:55,611 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 03:51:56,316 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-04 03:51:56,317 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: stop_status
+2024-08-04 03:51:56,317 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-04 03:52:01,592 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 03:52:06,593 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 03:52:11,316 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-04 03:52:11,317 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: stop_status
+2024-08-04 03:52:11,360 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-04 03:52:12,552 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 03:52:13,160 INFO    Thread-12 :12425 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_035140-nyllt780/files/config.yaml
+2024-08-04 03:52:17,755 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 03:52:22,755 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 03:52:26,316 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-04 03:52:26,317 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: stop_status
+2024-08-04 03:52:26,360 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-04 03:52:28,589 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 03:52:33,590 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 03:52:38,591 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 03:52:41,228 DEBUG   SystemMonitor:12425 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-08-04 03:52:41,230 DEBUG   SenderThread:12425 [sender.py:send():382] send: stats
+2024-08-04 03:52:41,316 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-04 03:52:41,317 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: stop_status
+2024-08-04 03:52:41,360 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-04 03:52:43,008 DEBUG   SenderThread:12425 [sender.py:send():382] send: config
+2024-08-04 03:52:43,008 DEBUG   SenderThread:12425 [sender.py:send():382] send: config
+2024-08-04 03:52:44,011 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 03:52:44,176 INFO    Thread-12 :12425 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_035140-nyllt780/files/output.log
+2024-08-04 03:52:44,726 DEBUG   SenderThread:12425 [sender.py:send():382] send: exit
+2024-08-04 03:52:44,726 INFO    SenderThread:12425 [sender.py:send_exit():589] handling exit code: 1
+2024-08-04 03:52:44,726 INFO    SenderThread:12425 [sender.py:send_exit():591] handling runtime: 63
+2024-08-04 03:52:44,741 INFO    SenderThread:12425 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-04 03:52:44,741 INFO    SenderThread:12425 [sender.py:send_exit():597] send defer
+2024-08-04 03:52:44,742 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:44,742 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 0
+2024-08-04 03:52:44,742 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:44,742 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 0
+2024-08-04 03:52:44,742 INFO    SenderThread:12425 [sender.py:transition_state():617] send defer: 1
+2024-08-04 03:52:44,742 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:44,742 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 1
+2024-08-04 03:52:44,742 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:44,742 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 1
+2024-08-04 03:52:44,742 INFO    SenderThread:12425 [sender.py:transition_state():617] send defer: 2
+2024-08-04 03:52:44,742 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:44,742 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 2
+2024-08-04 03:52:44,742 INFO    HandlerThread:12425 [system_monitor.py:finish():203] Stopping system monitor
+2024-08-04 03:52:44,743 DEBUG   SystemMonitor:12425 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-08-04 03:52:44,743 INFO    HandlerThread:12425 [interfaces.py:finish():202] Joined cpu monitor
+2024-08-04 03:52:44,743 DEBUG   SystemMonitor:12425 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-08-04 03:52:44,743 INFO    HandlerThread:12425 [interfaces.py:finish():202] Joined disk monitor
+2024-08-04 03:52:44,777 INFO    HandlerThread:12425 [interfaces.py:finish():202] Joined gpu monitor
+2024-08-04 03:52:44,777 INFO    HandlerThread:12425 [interfaces.py:finish():202] Joined memory monitor
+2024-08-04 03:52:44,777 INFO    HandlerThread:12425 [interfaces.py:finish():202] Joined network monitor
+2024-08-04 03:52:44,778 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:44,778 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 2
+2024-08-04 03:52:44,778 INFO    SenderThread:12425 [sender.py:transition_state():617] send defer: 3
+2024-08-04 03:52:44,778 DEBUG   SenderThread:12425 [sender.py:send():382] send: stats
+2024-08-04 03:52:44,778 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:44,778 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 3
+2024-08-04 03:52:44,778 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:44,779 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 3
+2024-08-04 03:52:44,779 INFO    SenderThread:12425 [sender.py:transition_state():617] send defer: 4
+2024-08-04 03:52:44,779 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:44,779 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 4
+2024-08-04 03:52:44,779 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:44,779 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 4
+2024-08-04 03:52:44,779 INFO    SenderThread:12425 [sender.py:transition_state():617] send defer: 5
+2024-08-04 03:52:44,779 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:44,779 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 5
+2024-08-04 03:52:44,779 DEBUG   SenderThread:12425 [sender.py:send():382] send: summary
+2024-08-04 03:52:44,780 INFO    SenderThread:12425 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-04 03:52:44,780 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:44,780 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 5
+2024-08-04 03:52:44,780 INFO    SenderThread:12425 [sender.py:transition_state():617] send defer: 6
+2024-08-04 03:52:44,780 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:44,781 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 6
+2024-08-04 03:52:44,781 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:44,781 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 6
+2024-08-04 03:52:44,781 INFO    SenderThread:12425 [sender.py:transition_state():617] send defer: 7
+2024-08-04 03:52:44,781 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 03:52:44,781 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:44,781 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 7
+2024-08-04 03:52:44,781 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:44,781 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 7
+2024-08-04 03:52:45,177 INFO    Thread-12 :12425 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_035140-nyllt780/files/config.yaml
+2024-08-04 03:52:45,177 INFO    Thread-12 :12425 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_035140-nyllt780/files/wandb-summary.json
+2024-08-04 03:52:45,726 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-04 03:52:46,178 INFO    Thread-12 :12425 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_035140-nyllt780/files/output.log
+2024-08-04 03:52:47,600 INFO    SenderThread:12425 [sender.py:transition_state():617] send defer: 8
+2024-08-04 03:52:47,600 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: poll_exit
+2024-08-04 03:52:47,600 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:47,601 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 8
+2024-08-04 03:52:47,601 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:47,601 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 8
+2024-08-04 03:52:47,601 INFO    SenderThread:12425 [job_builder.py:build():296] Attempting to build job artifact
+2024-08-04 03:52:47,602 INFO    SenderThread:12425 [job_builder.py:_get_source_type():426] is repo sourced job
+2024-08-04 03:52:47,616 INFO    SenderThread:12425 [job_builder.py:build():402] adding wandb-job metadata file
+2024-08-04 03:52:47,688 INFO    SenderThread:12425 [sender.py:transition_state():617] send defer: 9
+2024-08-04 03:52:47,689 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:47,689 DEBUG   SenderThread:12425 [sender.py:send():382] send: artifact
+2024-08-04 03:52:47,689 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 9
+2024-08-04 03:52:47,727 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-04 03:52:48,179 INFO    Thread-12 :12425 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_035140-nyllt780/files/output.log
+2024-08-04 03:52:48,575 INFO    SenderThread:12425 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA5MTk2NTkzOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA5MzUzODM4NQ==', 'versionIndex': 3}}}
+2024-08-04 03:52:48,575 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:48,575 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 9
+2024-08-04 03:52:48,575 INFO    SenderThread:12425 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-08-04 03:52:49,180 INFO    SenderThread:12425 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240804_035140-nyllt780/files
+2024-08-04 03:52:49,180 INFO    SenderThread:12425 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_035140-nyllt780/files/requirements.txt requirements.txt
+2024-08-04 03:52:49,180 INFO    SenderThread:12425 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_035140-nyllt780/files/config.yaml config.yaml
+2024-08-04 03:52:49,181 INFO    SenderThread:12425 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_035140-nyllt780/files/wandb-metadata.json wandb-metadata.json
+2024-08-04 03:52:49,182 INFO    SenderThread:12425 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_035140-nyllt780/files/wandb-summary.json wandb-summary.json
+2024-08-04 03:52:49,183 INFO    SenderThread:12425 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_035140-nyllt780/files/output.log output.log
+2024-08-04 03:52:49,185 INFO    SenderThread:12425 [sender.py:transition_state():617] send defer: 10
+2024-08-04 03:52:49,185 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: poll_exit
+2024-08-04 03:52:49,185 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:49,187 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 10
+2024-08-04 03:52:49,187 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:49,187 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 10
+2024-08-04 03:52:49,187 INFO    SenderThread:12425 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-04 03:52:49,580 INFO    wandb-upload_0:12425 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_035140-nyllt780/files/requirements.txt
+2024-08-04 03:52:49,719 INFO    wandb-upload_1:12425 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_035140-nyllt780/files/config.yaml
+2024-08-04 03:52:49,727 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-04 03:52:49,727 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: poll_exit
+2024-08-04 03:52:49,752 INFO    wandb-upload_2:12425 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_035140-nyllt780/files/wandb-summary.json
+2024-08-04 03:52:49,778 INFO    wandb-upload_3:12425 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_035140-nyllt780/files/output.log
+2024-08-04 03:52:49,978 INFO    Thread-11 (_thread_body):12425 [sender.py:transition_state():617] send defer: 11
+2024-08-04 03:52:49,978 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:49,979 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 11
+2024-08-04 03:52:49,979 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:49,979 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 11
+2024-08-04 03:52:49,979 INFO    SenderThread:12425 [file_pusher.py:join():178] waiting for file pusher
+2024-08-04 03:52:49,979 INFO    SenderThread:12425 [sender.py:transition_state():617] send defer: 12
+2024-08-04 03:52:49,979 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:49,979 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 12
+2024-08-04 03:52:49,979 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:49,979 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 12
+2024-08-04 03:52:49,979 INFO    SenderThread:12425 [file_stream.py:finish():595] file stream finish called
+2024-08-04 03:52:50,544 INFO    SenderThread:12425 [file_stream.py:finish():599] file stream finish is done
+2024-08-04 03:52:50,544 INFO    SenderThread:12425 [sender.py:transition_state():617] send defer: 13
+2024-08-04 03:52:50,545 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:50,545 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 13
+2024-08-04 03:52:50,545 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:50,545 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 13
+2024-08-04 03:52:50,545 INFO    SenderThread:12425 [sender.py:transition_state():617] send defer: 14
+2024-08-04 03:52:50,545 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 03:52:50,545 DEBUG   SenderThread:12425 [sender.py:send():382] send: final
+2024-08-04 03:52:50,545 INFO    HandlerThread:12425 [handler.py:handle_request_defer():172] handle defer: 14
+2024-08-04 03:52:50,545 DEBUG   SenderThread:12425 [sender.py:send():382] send: footer
+2024-08-04 03:52:50,546 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: defer
+2024-08-04 03:52:50,546 INFO    SenderThread:12425 [sender.py:send_request_defer():613] handle sender defer: 14
+2024-08-04 03:52:50,546 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-04 03:52:50,546 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: poll_exit
+2024-08-04 03:52:50,546 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-04 03:52:50,547 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: server_info
+2024-08-04 03:52:50,547 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: get_summary
+2024-08-04 03:52:50,547 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: poll_exit
+2024-08-04 03:52:50,547 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: sampled_history
+2024-08-04 03:52:50,547 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: server_info
+2024-08-04 03:52:50,548 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-04 03:52:50,549 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: job_info
+2024-08-04 03:52:50,716 DEBUG   SenderThread:12425 [sender.py:send_request():409] send_request: job_info
+2024-08-04 03:52:50,717 INFO    MainThread:12425 [wandb_run.py:_footer_history_summary_info():3866] rendering history
+2024-08-04 03:52:50,717 INFO    MainThread:12425 [wandb_run.py:_footer_history_summary_info():3898] rendering summary
+2024-08-04 03:52:50,717 INFO    MainThread:12425 [wandb_run.py:_footer_sync_info():3825] logging synced files
+2024-08-04 03:52:50,717 DEBUG   HandlerThread:12425 [handler.py:handle_request():146] handle_request: shutdown
+2024-08-04 03:52:50,717 INFO    HandlerThread:12425 [handler.py:finish():869] shutting down handler
+2024-08-04 03:52:51,549 INFO    WriterThread:12425 [datastore.py:close():296] close: /project/wandb/run-20240804_035140-nyllt780/run-nyllt780.wandb
+2024-08-04 03:52:51,717 INFO    SenderThread:12425 [sender.py:finish():1572] shutting down sender
+2024-08-04 03:52:51,717 INFO    SenderThread:12425 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-04 03:52:51,717 INFO    SenderThread:12425 [file_pusher.py:join():178] waiting for file pusher

wandb/run-20240804_035140-nyllt780/logs/debug.log

@@ -0,0 +1,30 @@
+2024-08-04 03:51:40,503 INFO    MainThread:12354 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
+2024-08-04 03:51:40,504 INFO    MainThread:12354 [wandb_setup.py:_flush():76] Configure stats pid to 12354
+2024-08-04 03:51:40,504 INFO    MainThread:12354 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings
+2024-08-04 03:51:40,504 INFO    MainThread:12354 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings
+2024-08-04 03:51:40,504 INFO    MainThread:12354 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tuny llama sample'}
+2024-08-04 03:51:40,504 INFO    MainThread:12354 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-08-04 03:51:40,504 INFO    MainThread:12354 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'}
+2024-08-04 03:51:40,504 INFO    MainThread:12354 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240804_035140-nyllt780/logs/debug.log
+2024-08-04 03:51:40,504 INFO    MainThread:12354 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240804_035140-nyllt780/logs/debug-internal.log
+2024-08-04 03:51:40,504 INFO    MainThread:12354 [wandb_init.py:init():566] calling init triggers
+2024-08-04 03:51:40,504 INFO    MainThread:12354 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
+config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 512, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-llama-sample_train_2024-08-04-03:51:30', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-llama-sample', 'save': '/work/llm_recipes/models/tiny-llama-sample', 'base_model': '/share/pretrained_lm/meta-llama/TinyLlama_v1.1', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 8, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-llama-sample', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32000, 'gradient_accumulation_steps': 40}
+2024-08-04 03:51:40,504 INFO    MainThread:12354 [wandb_init.py:init():616] starting backend
+2024-08-04 03:51:40,504 INFO    MainThread:12354 [wandb_init.py:init():620] setting up manager
+2024-08-04 03:51:40,509 INFO    MainThread:12354 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-08-04 03:51:40,510 INFO    MainThread:12354 [wandb_init.py:init():628] backend started and connected
+2024-08-04 03:51:40,515 INFO    MainThread:12354 [wandb_init.py:init():720] updated telemetry
+2024-08-04 03:51:40,654 INFO    MainThread:12354 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-08-04 03:51:41,132 INFO    MainThread:12354 [wandb_run.py:_on_init():2262] communicating current version
+2024-08-04 03:51:41,211 INFO    MainThread:12354 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-08-04 03:51:41,211 INFO    MainThread:12354 [wandb_init.py:init():804] starting run threads in backend
+2024-08-04 03:51:41,316 INFO    MainThread:12354 [wandb_run.py:_console_start():2241] atexit reg
+2024-08-04 03:51:41,317 INFO    MainThread:12354 [wandb_run.py:_redirect():2096] redirect: wrap_raw
+2024-08-04 03:51:41,317 INFO    MainThread:12354 [wandb_run.py:_redirect():2161] Wrapping output streams.
+2024-08-04 03:51:41,317 INFO    MainThread:12354 [wandb_run.py:_redirect():2186] Redirects installed.
+2024-08-04 03:51:41,318 INFO    MainThread:12354 [wandb_init.py:init():847] run started, returning control to user process
+2024-08-04 03:52:43,007 INFO    MainThread:12354 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'silu', 'hidden_size': 2048, 'model_type': 'llama', 'max_position_embeddings': 2048, 'num_attention_heads': 32, 'num_hidden_layers': 22, 'model_architecture': 'LlamaForCausalLM'}
+2024-08-04 03:52:43,008 INFO    MainThread:12354 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1}
+2024-08-04 03:52:51,718 WARNING MsgRouterThr:12354 [router.py:message_loop():77] message_loop has been closed

wandb/run-20240804_211947-niq3ake5/files/config.yaml

@@ -0,0 +1,335 @@
+wandb_version: 1
+
+sharding_strategy:
+  desc: null
+  value: FULL_SHARD
+checkpoint_type:
+  desc: null
+  value: LOCAL_STATE_DICT
+fsdp_activation_checkpointing:
+  desc: null
+  value: true
+fsdp_cpu_offload:
+  desc: null
+  value: false
+low_cpu_fsdp:
+  desc: null
+  value: false
+no_meta_device:
+  desc: null
+  value: false
+data_path:
+  desc: null
+  value: null
+split:
+  desc: null
+  value: 969, 30, 1
+train_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+valid_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+test_data_path:
+  desc: null
+  value:
+  - '4013541'
+  - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document
+data_cache_path:
+  desc: null
+  value: null
+vocab_size:
+  desc: null
+  value: null
+vocab_file:
+  desc: null
+  value: null
+merge_file:
+  desc: null
+  value: null
+seq_length:
+  desc: null
+  value: 512
+num_workers:
+  desc: null
+  value: 2
+tokenizer_type:
+  desc: null
+  value: Llama2Tokenizer
+tokenizer_model:
+  desc: null
+  value: /share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model
+reset_position_ids:
+  desc: null
+  value: false
+reset_attention_mask:
+  desc: null
+  value: false
+eod_mask_loss:
+  desc: null
+  value: false
+retro_return_doc_ids:
+  desc: null
+  value: false
+short_seq_prob:
+  desc: null
+  value: 0.1
+vocab_extra_ids:
+  desc: null
+  value: 0
+seed:
+  desc: null
+  value: 1234
+use_mpi:
+  desc: null
+  value: false
+wandb_entity:
+  desc: null
+  value: iwakawa-koichi-q5-tohoku-nlp6723
+wandb_name:
+  desc: null
+  value: tiny-llama_train_2024-08-04-21:19:16
+wandb_project:
+  desc: null
+  value: llm_tutorial
+quantization:
+  desc: null
+  value: false
+use_freeze_layers:
+  desc: null
+  value: false
+freeze_layers:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+mixed_precision:
+  desc: null
+  value: true
+param_dtype:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: /work/llm_recipes/models/tiny-llama
+save:
+  desc: null
+  value: /work/llm_recipes/models/tiny-llama
+base_model:
+  desc: null
+  value: /share/pretrained_lm/meta-llama/TinyLlama_v1.1
+use_better_transformer:
+  desc: null
+  value: false
+grad_clip_norm:
+  desc: null
+  value: 1.0
+eval_interval:
+  desc: null
+  value: 200
+save_interval:
+  desc: null
+  value: 200
+eval_iters:
+  desc: null
+  value: 10
+optimizer:
+  desc: null
+  value: adam
+lr:
+  desc: null
+  value: 2.0e-05
+lr_decay_style:
+  desc: null
+  value: cosine
+lr_decay_iters:
+  desc: null
+  value: 2000
+lr_warmup_iters:
+  desc: null
+  value: 500
+min_lr:
+  desc: null
+  value: 1.0e-06
+train_iters:
+  desc: null
+  value: 2000
+train_samples:
+  desc: null
+  value: null
+global_batch_size:
+  desc: null
+  value: 320
+micro_batch_size:
+  desc: null
+  value: 8
+make_vocab_size_divisible_by:
+  desc: null
+  value: 128
+sliding_window_size:
+  desc: null
+  value: 4096
+skip_batch:
+  desc: null
+  value: null
+no_save_optimizer_state:
+  desc: null
+  value: false
+continual_pretraining:
+  desc: null
+  value: false
+instruction_tuning:
+  desc: null
+  value: false
+direct_preference_optimization:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.1
+hidden_dropout:
+  desc: null
+  value: 0.1
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_eps:
+  desc: null
+  value: 1.0e-06
+hf_transformer_model_dir:
+  desc: null
+  value: null
+instruction_train_data_path:
+  desc: null
+  value: null
+instruction_valid_data_path:
+  desc: null
+  value: null
+epoch:
+  desc: null
+  value: null
+instruction_dataset_size:
+  desc: null
+  value: null
+save_sampler_state:
+  desc: null
+  value: false
+label_smoothing:
+  desc: null
+  value: 0.0
+save_n_checkpoints:
+  desc: null
+  value: 10
+hf_repo_id:
+  desc: null
+  value: koichi12/tiny-llama
+create_public_hf_repo:
+  desc: null
+  value: false
+upload_all_checkpoints_to_hf:
+  desc: null
+  value: false
+hf_upload_retry_limit:
+  desc: null
+  value: 2
+exit_duration_in_mins:
+  desc: null
+  value: null
+source_key:
+  desc: null
+  value: null
+target_key:
+  desc: null
+  value: null
+attn_implementation:
+  desc: null
+  value: flash_attention_2
+efficient_instruction_tuning:
+  desc: null
+  value: false
+remove_padding_masking:
+  desc: null
+  value: false
+save_start_iter:
+  desc: null
+  value: null
+rank:
+  desc: null
+  value: 0
+world_size:
+  desc: null
+  value: 1
+padded_vocab_size:
+  desc: null
+  value: 32000
+gradient_accumulation_steps:
+  desc: null
+  value: 40
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.43.3
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1722773987.17106
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      2:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.43.3
+      8:
+      - 5
+      13: linux-x86_64
+activation_function:
+  desc: null
+  value: silu
+hidden_size:
+  desc: null
+  value: 2048
+model_type:
+  desc: null
+  value: llama
+max_position_embeddings:
+  desc: null
+  value: 2048
+num_attention_heads:
+  desc: null
+  value: 32
+num_hidden_layers:
+  desc: null
+  value: 22
+model_architecture:
+  desc: null
+  value: LlamaForCausalLM

wandb/run-20240804_211947-niq3ake5/files/output.log

@@ -0,0 +1,135 @@
+Created Hugging Face repository with ID koichi12/tiny-llama.
+Clearing GPU cache for all ranks
+--> Running with torch torch_distributed debug set to detail
+File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt
+File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt
+File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt
+No checkpoint found in /work/llm_recipes/models/tiny-llama, skipping model loading
+--> Model /share/pretrained_lm/meta-llama/TinyLlama_v1.1
+--> /share/pretrained_lm/meta-llama/TinyLlama_v1.1 has 1100.048384 Million params
+You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)`
+Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaModel is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)`
+/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1.
+  warnings.warn(
+BFloat16 enabled for mixed precision - using bfSixteen policy
+--> applying fsdp activation checkpointing...
+ > datasets target sizes (minimum size):
+    train:      640000
+    validation: 35200
+    test:       3200
+> building train, validation, and test datasets for GPT ...
+> finished creating GPT datasets ...
+File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt
+No checkpoint found in /work/llm_recipes/models/tiny-llama, skipping optimizer loading
+File not found: /work/llm_recipes/models/tiny-llama/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/tiny-llama/latest_iteration.txt
+model info: FullyShardedDataParallel(
+  (_fsdp_wrapped_module): LlamaForCausalLM(
+    (model): LlamaModel(
+      (embed_tokens): Embedding(32000, 2048)
+      (layers): ModuleList(
+        (0-21): 22 x FullyShardedDataParallel(
+          (_fsdp_wrapped_module): CheckpointWrapper(
+            (_checkpoint_wrapped_module): LlamaDecoderLayer(
+              (self_attn): LlamaFlashAttention2(
+                (q_proj): Linear(in_features=2048, out_features=2048, bias=False)
+                (k_proj): Linear(in_features=2048, out_features=256, bias=False)
+                (v_proj): Linear(in_features=2048, out_features=256, bias=False)
+                (o_proj): Linear(in_features=2048, out_features=2048, bias=False)
+                (rotary_emb): LlamaRotaryEmbedding()
+              )
+              (mlp): LlamaMLP(
+                (gate_proj): Linear(in_features=2048, out_features=5632, bias=False)
+                (up_proj): Linear(in_features=2048, out_features=5632, bias=False)
+                (down_proj): Linear(in_features=5632, out_features=2048, bias=False)
+                (act_fn): SiLU()
+              )
+              (input_layernorm): LlamaRMSNorm()
+              (post_attention_layernorm): LlamaRMSNorm()
+            )
+          )
+        )
+      )
+      (norm): LlamaRMSNorm()
+      (rotary_emb): LlamaRotaryEmbedding()
+    )
+    (lm_head): Linear(in_features=2048, out_features=32000, bias=False)
+  )
+)
+model config: LlamaConfig {
+  "_name_or_path": "/share/pretrained_lm/meta-llama/TinyLlama_v1.1",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5632,
+  "label_smoothing": 0.0,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 22,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.43.3",
+  "use_cache": false,
+  "vocab_size": 32000
+}
+Let split = None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+Traceback (most recent call last):
+  File "/project/examples/finetuning.py", line 13, in <module>
+    main()
+  File "/project/src/llama_recipes/finetuning.py", line 281, in main
+    train(
+  File "/project/src/llama_recipes/utils/train_utils.py", line 104, in train
+    batch = next(train_dataloader)
+  File "/project/src/llama_recipes/utils/train_utils.py", line 24, in cyclic_iter
+    for x in iter:
+  File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py", line 631, in __next__
+    data = self._next_data()
+  File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py", line 1346, in _next_data
+    return self._process_data(data)
+  File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py", line 1372, in _process_data
+    data.reraise()
+  File "/usr/local/lib/python3.10/dist-packages/torch/_utils.py", line 705, in reraise
+    raise exception
+RuntimeError: Caught RuntimeError in DataLoader worker process 0.
+Original Traceback (most recent call last):
+  File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/worker.py", line 308, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/fetch.py", line 54, in fetch
+    return self.collate_fn(data)
+  File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 277, in default_collate
+    return collate(batch, collate_fn_map=default_collate_fn_map)
+  File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 129, in collate
+    return elem_type({key: collate([d[key] for d in batch], collate_fn_map=collate_fn_map) for key in elem})
+  File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 129, in <dictcomp>
+    return elem_type({key: collate([d[key] for d in batch], collate_fn_map=collate_fn_map) for key in elem})
+  File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 121, in collate
+    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)
+  File "/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py", line 174, in collate_tensor_fn
+    return torch.stack(batch, 0, out=out)
+RuntimeError: stack expects each tensor to be equal size, but got [513] at entry 0 and [543] at entry 1

wandb/run-20240804_211947-niq3ake5/files/requirements.txt

@@ -0,0 +1,271 @@
+absl-py==2.1.0
+accelerate==0.33.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+apex==0.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+beautifulsoup4==4.12.3
+bleach==6.1.0
+blis==0.7.11
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+cmake==3.28.1
+colorama==0.4.6
+comm==0.2.1
+confection==0.1.4
+contourpy==1.2.0
+cubinlinker==0.3.0+2.g405ac64
+cuda-python==12.3.0rc4+9.gdb8c48a.dirty
+cudf==23.12.0
+cugraph-dgl==23.12.0
+cugraph-service-client==23.12.0
+cugraph-service-server==23.12.0
+cugraph==23.12.0
+cuml==23.12.0
+cupy-cuda12x==12.3.0
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.8
+dask-cuda==23.12.0
+dask-cudf==23.12.0
+dask==2023.11.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+distributed==2023.11.0
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+einops==0.7.0
+exceptiongroup==1.2.0
+execnet==2.0.2
+executing==2.0.1
+expecttest==0.1.3
+fastjsonschema==2.19.1
+fastrlock==0.8.2
+filelock==3.13.1
+flash-attn==2.4.2
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2023.12.2
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.27.0
+graphsurgeon==0.4.6
+grpcio==1.60.1
+huggingface-hub==0.24.5
+hypothesis==5.35.1
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+intel-openmp==2021.4.0
+ipadic==1.0.0
+ipykernel==6.29.2
+ipython-genutils==0.2.0
+ipython==8.21.0
+jedi==0.19.1
+jinja2==3.1.3
+joblib==1.3.2
+json5==0.9.14
+jsonnet==0.19.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.21.1
+jupyter-client==8.6.0
+jupyter-core==5.7.1
+jupyter-tensorboard==0.2.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==1.2.0
+jupyterlab==2.3.2
+jupytext==1.16.1
+kiwisolver==1.4.5
+langcodes==3.3.0
+lazy-loader==0.3
+librosa==0.10.1
+llvmlite==0.40.1
+locket==1.0.0
+logzero==1.7.0
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown==3.5.2
+markupsafe==2.1.4
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mecab-python3==1.0.6
+mistune==3.0.2
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mkl==2021.1.1
+mock==5.1.0
+more-itertools==9.1.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+murmurhash==1.0.10
+nbclient==0.9.0
+nbconvert==7.16.0
+nbformat==5.9.2
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.4.10
+numba==0.57.1+1.g1ff679645
+numpy==1.24.4
+nvfuser==0.1.4a0+d0bb811
+nvidia-dali-cuda120==1.34.0
+nvidia-pyindex==1.0.9
+nvtx==0.2.5
+oauthlib==3.2.2
+onnx==1.15.0rc2
+opencv==4.7.0
+optree==0.10.0
+packaging==23.2
+pandas==1.5.3
+pandocfilters==1.5.1
+parso==0.8.3
+partd==1.4.1
+peft==0.11.1
+pexpect==4.9.0
+pillow==10.2.0
+pip==24.0
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+polygraphy==0.49.4
+pooch==1.8.0
+portalocker==2.10.1
+preshed==3.0.9
+prettytable==3.9.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.24.4
+psutil==5.9.4
+ptxcompiler==0.8.1+2.g0d406d6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==14.0.1.dev0+gba5374836.d20240125
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pybind11-global==2.11.1
+pybind11==2.11.1
+pycocotools==2.0+nv0.8.0
+pycparser==2.21
+pydantic-core==2.16.2
+pydantic==2.6.1
+pygments==2.17.2
+pylibcugraph==23.12.0
+pylibcugraphops==23.12.0
+pylibraft==23.12.0
+pynvml==11.4.1
+pyparsing==3.1.1
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==13.0
+pytest-shard==0.1.2
+pytest-xdist==3.5.0
+pytest==8.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-hostlist==1.23.0
+pytorch-quantization==2.1.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.2
+raft-dask==23.12.0
+rapids-dask-dependency==23.12.1
+referencing==0.33.0
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests==2.31.0
+rich==13.7.0
+rmm==23.12.0
+rpds-py==0.17.1
+rsa==4.9
+sacrebleu==2.4.0
+safetensors==0.4.3
+scikit-learn==1.2.0
+scipy==1.12.0
+send2trash==1.8.2
+sentencepiece==0.1.99
+sentry-sdk==2.12.0
+setproctitle==1.3.3
+setuptools==68.2.2
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sphinx-glpi-theme==0.6
+srsly==2.4.8
+stack-data==0.6.3
+sympy==1.12
+tabulate==0.9.0
+tbb==2021.11.0
+tblib==3.0.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.9.0
+tensorrt==8.6.3
+terminado==0.18.0
+termplotlib==0.3.9
+thinc==8.2.3
+threadpoolctl==3.2.0
+thriftpy2==0.4.17
+tinycss2==1.2.1
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.1
+torch-tensorrt==2.3.0a0
+torch==2.3.0a0+ebedce2
+torchdata==0.7.1a0
+torchtext==0.17.0a0
+torchvision==0.18.0a0
+tornado==6.4
+tqdm==4.66.1
+traitlets==5.9.0
+transformer-engine==1.3.0+5b90b7f
+transformers==4.43.3
+treelite-runtime==3.9.1
+treelite==3.9.1
+triton==2.2.0+e28a256
+typer==0.9.0
+types-dataclasses==0.6.6
+typing-extensions==4.9.0
+ucx-py==0.35.0
+uff==0.6.9
+ujson==5.8.0
+urllib3==1.26.18
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.3.4
+webencodings==0.5.1
+werkzeug==3.0.1
+wheel==0.42.0
+xdoctest==1.0.2
+xgboost==1.7.6
+yarl==1.9.4
+zict==3.0.0
+zipp==3.17.0

wandb/run-20240804_211947-niq3ake5/files/wandb-metadata.json

@@ -0,0 +1,215 @@
+{
+    "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-08-04T12:19:47.940599",
+    "startedAt": "2024-08-04T12:19:47.157671",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--seq-length",
+        "512",
+        "--sliding-window-size",
+        "4096",
+        "--micro-batch-size",
+        "8",
+        "--global-batch-size",
+        "320",
+        "--train-iters",
+        "2000",
+        "--tokenizer-type",
+        "Llama2Tokenizer",
+        "--tokenizer-model",
+        "/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model",
+        "--train-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--valid-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--test-data-path",
+        "4013541",
+        "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document",
+        "--lr",
+        "2e-5",
+        "--min-lr",
+        "1e-6",
+        "--lr-decay-style",
+        "cosine",
+        "--lr-warmup-iters",
+        "500",
+        "--lr-decay-iters",
+        "2000",
+        "--weight-decay",
+        "0.1",
+        "--grad-clip-norm",
+        "1.0",
+        "--optimizer",
+        "adam",
+        "--adam-beta1",
+        "0.9",
+        "--adam-beta2",
+        "0.95",
+        "--adam-eps",
+        "1e-6",
+        "--save-interval",
+        "200",
+        "--eval-interval",
+        "200",
+        "--eval-iters",
+        "10",
+        "--bf16",
+        "--mixed-precision",
+        "--base-model",
+        "/share/pretrained_lm/meta-llama/TinyLlama_v1.1",
+        "--save",
+        "/work/llm_recipes/models/tiny-llama",
+        "--load",
+        "/work/llm_recipes/models/tiny-llama",
+        "--fsdp-activation-checkpointing",
+        "--sharding-strategy",
+        "FULL_SHARD",
+        "--checkpoint-type",
+        "LOCAL_STATE_DICT",
+        "--save-n-checkpoints",
+        "10",
+        "--hf-upload-retry-limit",
+        "2",
+        "--hf-repo-id",
+        "koichi12/tiny-llama",
+        "--wandb-entity",
+        "iwakawa-koichi-q5-tohoku-nlp6723",
+        "--wandb-project",
+        "llm_tutorial",
+        "--wandb-name",
+        "tiny-llama_train_2024-08-04-21:19:16"
+    ],
+    "state": "running",
+    "program": "/project/examples/finetuning.py",
+    "codePathLocal": "examples/finetuning.py",
+    "codePath": "examples/finetuning.py",
+    "git": {
+        "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git",
+        "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea"
+    },
+    "email": null,
+    "root": "/project",
+    "host": "gpu-koiwa-00",
+    "username": "koiwa",
+    "executable": "/usr/bin/python",
+    "cpu_count": 18,
+    "cpu_count_logical": 18,
+    "cpu_freq": {
+        "current": 2400.044999999999,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.045,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 0.0625,
+            "used": 1.1444091796875e-05
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 56.48782730102539
+    }
+}

wandb/run-20240804_211947-niq3ake5/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 67}}

wandb/run-20240804_211947-niq3ake5/logs/debug-internal.log

@@ -0,0 +1,213 @@
+2024-08-04 21:19:47,171 INFO    StreamThr :10096 [internal.py:wandb_internal():86] W&B internal server running at pid: 10096, started at: 2024-08-04 21:19:47.170590
+2024-08-04 21:19:47,173 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status
+2024-08-04 21:19:47,176 INFO    WriterThread:10096 [datastore.py:open_for_write():87] open: /project/wandb/run-20240804_211947-niq3ake5/run-niq3ake5.wandb
+2024-08-04 21:19:47,177 DEBUG   SenderThread:10096 [sender.py:send():382] send: header
+2024-08-04 21:19:47,316 DEBUG   SenderThread:10096 [sender.py:send():382] send: run
+2024-08-04 21:19:47,822 INFO    SenderThread:10096 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240804_211947-niq3ake5/files
+2024-08-04 21:19:47,822 INFO    SenderThread:10096 [sender.py:_start_run_threads():1136] run started: niq3ake5 with start time 1722773987.17106
+2024-08-04 21:19:47,827 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: check_version
+2024-08-04 21:19:47,827 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: check_version
+2024-08-04 21:19:47,917 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: run_start
+2024-08-04 21:19:47,923 DEBUG   HandlerThread:10096 [system_info.py:__init__():27] System info init
+2024-08-04 21:19:47,923 DEBUG   HandlerThread:10096 [system_info.py:__init__():42] System info init done
+2024-08-04 21:19:47,923 INFO    HandlerThread:10096 [system_monitor.py:start():194] Starting system monitor
+2024-08-04 21:19:47,923 INFO    SystemMonitor:10096 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-08-04 21:19:47,924 INFO    HandlerThread:10096 [system_monitor.py:probe():214] Collecting system info
+2024-08-04 21:19:47,924 INFO    SystemMonitor:10096 [interfaces.py:start():190] Started cpu monitoring
+2024-08-04 21:19:47,924 INFO    SystemMonitor:10096 [interfaces.py:start():190] Started disk monitoring
+2024-08-04 21:19:47,925 INFO    SystemMonitor:10096 [interfaces.py:start():190] Started gpu monitoring
+2024-08-04 21:19:47,926 INFO    SystemMonitor:10096 [interfaces.py:start():190] Started memory monitoring
+2024-08-04 21:19:47,927 INFO    SystemMonitor:10096 [interfaces.py:start():190] Started network monitoring
+2024-08-04 21:19:47,940 DEBUG   HandlerThread:10096 [system_info.py:probe():151] Probing system
+2024-08-04 21:19:47,942 DEBUG   HandlerThread:10096 [system_info.py:_probe_git():136] Probing git
+2024-08-04 21:19:47,954 DEBUG   HandlerThread:10096 [system_info.py:_probe_git():144] Probing git done
+2024-08-04 21:19:47,954 DEBUG   HandlerThread:10096 [system_info.py:probe():199] Probing system done
+2024-08-04 21:19:47,954 DEBUG   HandlerThread:10096 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-04T12:19:47.940599', 'startedAt': '2024-08-04T12:19:47.157671', 'docker': None, 'cuda': None, 'args': ('--seq-length', '512', '--sliding-window-size', '4096', '--micro-batch-size', '8', '--global-batch-size', '320', '--train-iters', '2000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '2000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/meta-llama/TinyLlama_v1.1', '--save', '/work/llm_recipes/models/tiny-llama', '--load', '/work/llm_recipes/models/tiny-llama', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-llama', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-llama_train_2024-08-04-21:19:16'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.044999999999, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}, {'current': 2400.045, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}}
+2024-08-04 21:19:47,954 INFO    HandlerThread:10096 [system_monitor.py:probe():224] Finished collecting system info
+2024-08-04 21:19:47,954 INFO    HandlerThread:10096 [system_monitor.py:probe():227] Publishing system info
+2024-08-04 21:19:47,955 INFO    HandlerThread:10096 [system_monitor.py:probe():229] Finished publishing system info
+2024-08-04 21:19:47,982 DEBUG   SenderThread:10096 [sender.py:send():382] send: files
+2024-08-04 21:19:47,982 INFO    SenderThread:10096 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now
+2024-08-04 21:19:47,991 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: python_packages
+2024-08-04 21:19:47,992 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: python_packages
+2024-08-04 21:19:47,992 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-04 21:19:47,992 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-04 21:19:47,993 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: stop_status
+2024-08-04 21:19:48,264 DEBUG   SenderThread:10096 [sender.py:send():382] send: telemetry
+2024-08-04 21:19:48,653 INFO    wandb-upload_0:10096 [upload_job.py:push():131] Uploaded file /tmp/tmpc_z53slvwandb/somhprnl-wandb-metadata.json
+2024-08-04 21:19:48,823 INFO    Thread-12 :10096 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_211947-niq3ake5/files/requirements.txt
+2024-08-04 21:19:48,824 INFO    Thread-12 :10096 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_211947-niq3ake5/files/wandb-metadata.json
+2024-08-04 21:19:52,265 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 21:19:53,826 INFO    Thread-12 :10096 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_211947-niq3ake5/files/output.log
+2024-08-04 21:19:55,827 INFO    Thread-12 :10096 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_211947-niq3ake5/files/output.log
+2024-08-04 21:19:57,441 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 21:19:59,829 INFO    Thread-12 :10096 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_211947-niq3ake5/files/output.log
+2024-08-04 21:20:02,739 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 21:20:02,991 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-04 21:20:02,991 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-04 21:20:02,992 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: stop_status
+2024-08-04 21:20:08,241 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 21:20:13,242 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 21:20:17,991 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-04 21:20:17,991 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: stop_status
+2024-08-04 21:20:18,032 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-04 21:20:18,266 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 21:20:18,841 INFO    Thread-12 :10096 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_211947-niq3ake5/files/config.yaml
+2024-08-04 21:20:23,460 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 21:20:28,461 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 21:20:32,991 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-04 21:20:32,991 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: stop_status
+2024-08-04 21:20:33,032 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-04 21:20:34,171 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 21:20:39,171 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 21:20:44,172 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 21:20:47,927 DEBUG   SystemMonitor:10096 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-08-04 21:20:47,929 DEBUG   SenderThread:10096 [sender.py:send():382] send: stats
+2024-08-04 21:20:47,991 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-04 21:20:47,991 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: stop_status
+2024-08-04 21:20:48,032 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-04 21:20:49,259 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 21:20:53,862 INFO    Thread-12 :10096 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_211947-niq3ake5/files/output.log
+2024-08-04 21:20:54,334 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 21:20:54,437 DEBUG   SenderThread:10096 [sender.py:send():382] send: config
+2024-08-04 21:20:54,437 DEBUG   SenderThread:10096 [sender.py:send():382] send: config
+2024-08-04 21:20:55,009 DEBUG   SenderThread:10096 [sender.py:send():382] send: exit
+2024-08-04 21:20:55,009 INFO    SenderThread:10096 [sender.py:send_exit():589] handling exit code: 1
+2024-08-04 21:20:55,009 INFO    SenderThread:10096 [sender.py:send_exit():591] handling runtime: 67
+2024-08-04 21:20:55,010 INFO    SenderThread:10096 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-04 21:20:55,011 INFO    SenderThread:10096 [sender.py:send_exit():597] send defer
+2024-08-04 21:20:55,011 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:55,011 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 0
+2024-08-04 21:20:55,011 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:55,011 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 0
+2024-08-04 21:20:55,011 INFO    SenderThread:10096 [sender.py:transition_state():617] send defer: 1
+2024-08-04 21:20:55,011 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:55,011 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 1
+2024-08-04 21:20:55,011 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:55,011 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 1
+2024-08-04 21:20:55,012 INFO    SenderThread:10096 [sender.py:transition_state():617] send defer: 2
+2024-08-04 21:20:55,012 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:55,012 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 2
+2024-08-04 21:20:55,012 INFO    HandlerThread:10096 [system_monitor.py:finish():203] Stopping system monitor
+2024-08-04 21:20:55,012 DEBUG   SystemMonitor:10096 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-08-04 21:20:55,012 INFO    HandlerThread:10096 [interfaces.py:finish():202] Joined cpu monitor
+2024-08-04 21:20:55,012 DEBUG   SystemMonitor:10096 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-08-04 21:20:55,012 INFO    HandlerThread:10096 [interfaces.py:finish():202] Joined disk monitor
+2024-08-04 21:20:55,046 INFO    HandlerThread:10096 [interfaces.py:finish():202] Joined gpu monitor
+2024-08-04 21:20:55,047 INFO    HandlerThread:10096 [interfaces.py:finish():202] Joined memory monitor
+2024-08-04 21:20:55,047 INFO    HandlerThread:10096 [interfaces.py:finish():202] Joined network monitor
+2024-08-04 21:20:55,047 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:55,047 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 2
+2024-08-04 21:20:55,047 INFO    SenderThread:10096 [sender.py:transition_state():617] send defer: 3
+2024-08-04 21:20:55,047 DEBUG   SenderThread:10096 [sender.py:send():382] send: stats
+2024-08-04 21:20:55,047 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:55,048 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 3
+2024-08-04 21:20:55,048 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:55,048 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 3
+2024-08-04 21:20:55,048 INFO    SenderThread:10096 [sender.py:transition_state():617] send defer: 4
+2024-08-04 21:20:55,048 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:55,048 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 4
+2024-08-04 21:20:55,048 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:55,048 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 4
+2024-08-04 21:20:55,048 INFO    SenderThread:10096 [sender.py:transition_state():617] send defer: 5
+2024-08-04 21:20:55,048 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:55,048 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 5
+2024-08-04 21:20:55,049 DEBUG   SenderThread:10096 [sender.py:send():382] send: summary
+2024-08-04 21:20:55,050 INFO    SenderThread:10096 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-04 21:20:55,050 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:55,050 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 5
+2024-08-04 21:20:55,050 INFO    SenderThread:10096 [sender.py:transition_state():617] send defer: 6
+2024-08-04 21:20:55,050 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:55,050 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 6
+2024-08-04 21:20:55,050 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:55,050 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 6
+2024-08-04 21:20:55,053 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: status_report
+2024-08-04 21:20:55,265 INFO    SenderThread:10096 [sender.py:transition_state():617] send defer: 7
+2024-08-04 21:20:55,265 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:55,265 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 7
+2024-08-04 21:20:55,265 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:55,265 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 7
+2024-08-04 21:20:55,512 INFO    SenderThread:10096 [sender.py:transition_state():617] send defer: 8
+2024-08-04 21:20:55,513 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:55,513 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 8
+2024-08-04 21:20:55,513 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:55,513 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 8
+2024-08-04 21:20:55,513 INFO    SenderThread:10096 [job_builder.py:build():296] Attempting to build job artifact
+2024-08-04 21:20:55,514 INFO    SenderThread:10096 [job_builder.py:_get_source_type():426] is repo sourced job
+2024-08-04 21:20:55,528 INFO    SenderThread:10096 [job_builder.py:build():402] adding wandb-job metadata file
+2024-08-04 21:20:55,537 INFO    SenderThread:10096 [sender.py:transition_state():617] send defer: 9
+2024-08-04 21:20:55,537 DEBUG   SenderThread:10096 [sender.py:send():382] send: artifact
+2024-08-04 21:20:55,537 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:55,539 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 9
+2024-08-04 21:20:55,864 INFO    Thread-12 :10096 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_211947-niq3ake5/files/config.yaml
+2024-08-04 21:20:55,864 INFO    Thread-12 :10096 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_211947-niq3ake5/files/output.log
+2024-08-04 21:20:55,864 INFO    Thread-12 :10096 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_211947-niq3ake5/files/wandb-summary.json
+2024-08-04 21:20:56,009 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-04 21:20:57,540 INFO    SenderThread:10096 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA5MTk2NTkzOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA5MzUzODM4NQ==', 'versionIndex': 3}}}
+2024-08-04 21:20:57,540 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:57,540 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 9
+2024-08-04 21:20:57,540 INFO    SenderThread:10096 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-08-04 21:20:57,865 INFO    SenderThread:10096 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240804_211947-niq3ake5/files
+2024-08-04 21:20:57,865 INFO    SenderThread:10096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_211947-niq3ake5/files/requirements.txt requirements.txt
+2024-08-04 21:20:57,865 INFO    SenderThread:10096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_211947-niq3ake5/files/config.yaml config.yaml
+2024-08-04 21:20:57,867 INFO    SenderThread:10096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_211947-niq3ake5/files/wandb-metadata.json wandb-metadata.json
+2024-08-04 21:20:57,867 INFO    SenderThread:10096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_211947-niq3ake5/files/wandb-summary.json wandb-summary.json
+2024-08-04 21:20:57,869 INFO    SenderThread:10096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_211947-niq3ake5/files/output.log output.log
+2024-08-04 21:20:57,869 INFO    SenderThread:10096 [sender.py:transition_state():617] send defer: 10
+2024-08-04 21:20:57,870 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: poll_exit
+2024-08-04 21:20:57,872 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:57,872 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 10
+2024-08-04 21:20:57,872 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:57,872 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 10
+2024-08-04 21:20:57,872 INFO    SenderThread:10096 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-04 21:20:58,009 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-04 21:20:58,010 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: poll_exit
+2024-08-04 21:20:58,272 INFO    wandb-upload_1:10096 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_211947-niq3ake5/files/config.yaml
+2024-08-04 21:20:58,376 INFO    wandb-upload_0:10096 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_211947-niq3ake5/files/requirements.txt
+2024-08-04 21:20:58,453 INFO    wandb-upload_2:10096 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_211947-niq3ake5/files/wandb-summary.json
+2024-08-04 21:20:58,476 INFO    wandb-upload_3:10096 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_211947-niq3ake5/files/output.log
+2024-08-04 21:20:58,677 INFO    Thread-11 (_thread_body):10096 [sender.py:transition_state():617] send defer: 11
+2024-08-04 21:20:58,677 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:58,677 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 11
+2024-08-04 21:20:58,677 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:58,677 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 11
+2024-08-04 21:20:58,677 INFO    SenderThread:10096 [file_pusher.py:join():178] waiting for file pusher
+2024-08-04 21:20:58,677 INFO    SenderThread:10096 [sender.py:transition_state():617] send defer: 12
+2024-08-04 21:20:58,677 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:58,677 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 12
+2024-08-04 21:20:58,678 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:58,678 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 12
+2024-08-04 21:20:58,678 INFO    SenderThread:10096 [file_stream.py:finish():595] file stream finish called
+2024-08-04 21:20:58,860 INFO    SenderThread:10096 [file_stream.py:finish():599] file stream finish is done
+2024-08-04 21:20:58,860 INFO    SenderThread:10096 [sender.py:transition_state():617] send defer: 13
+2024-08-04 21:20:58,860 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:58,860 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 13
+2024-08-04 21:20:58,860 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:58,860 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 13
+2024-08-04 21:20:58,860 INFO    SenderThread:10096 [sender.py:transition_state():617] send defer: 14
+2024-08-04 21:20:58,861 DEBUG   SenderThread:10096 [sender.py:send():382] send: final
+2024-08-04 21:20:58,861 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: defer
+2024-08-04 21:20:58,861 DEBUG   SenderThread:10096 [sender.py:send():382] send: footer
+2024-08-04 21:20:58,861 INFO    HandlerThread:10096 [handler.py:handle_request_defer():172] handle defer: 14
+2024-08-04 21:20:58,861 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: defer
+2024-08-04 21:20:58,861 INFO    SenderThread:10096 [sender.py:send_request_defer():613] handle sender defer: 14
+2024-08-04 21:20:58,861 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-04 21:20:58,862 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: poll_exit
+2024-08-04 21:20:58,862 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-04 21:20:58,862 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: poll_exit
+2024-08-04 21:20:58,862 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: server_info
+2024-08-04 21:20:58,863 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: server_info
+2024-08-04 21:20:58,864 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: get_summary
+2024-08-04 21:20:58,864 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: sampled_history
+2024-08-04 21:20:58,864 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-04 21:20:58,865 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: job_info
+2024-08-04 21:20:59,033 DEBUG   SenderThread:10096 [sender.py:send_request():409] send_request: job_info
+2024-08-04 21:20:59,033 INFO    MainThread:10096 [wandb_run.py:_footer_history_summary_info():3866] rendering history
+2024-08-04 21:20:59,033 INFO    MainThread:10096 [wandb_run.py:_footer_history_summary_info():3898] rendering summary
+2024-08-04 21:20:59,033 INFO    MainThread:10096 [wandb_run.py:_footer_sync_info():3825] logging synced files
+2024-08-04 21:20:59,033 DEBUG   HandlerThread:10096 [handler.py:handle_request():146] handle_request: shutdown
+2024-08-04 21:20:59,033 INFO    HandlerThread:10096 [handler.py:finish():869] shutting down handler
+2024-08-04 21:20:59,865 INFO    WriterThread:10096 [datastore.py:close():296] close: /project/wandb/run-20240804_211947-niq3ake5/run-niq3ake5.wandb
+2024-08-04 21:21:00,033 INFO    SenderThread:10096 [sender.py:finish():1572] shutting down sender
+2024-08-04 21:21:00,033 INFO    SenderThread:10096 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-04 21:21:00,033 INFO    SenderThread:10096 [file_pusher.py:join():178] waiting for file pusher

wandb/run-20240804_211947-niq3ake5/logs/debug.log

@@ -0,0 +1,30 @@
+2024-08-04 21:19:47,163 INFO    MainThread:10025 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
+2024-08-04 21:19:47,163 INFO    MainThread:10025 [wandb_setup.py:_flush():76] Configure stats pid to 10025
+2024-08-04 21:19:47,164 INFO    MainThread:10025 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings
+2024-08-04 21:19:47,164 INFO    MainThread:10025 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings
+2024-08-04 21:19:47,164 INFO    MainThread:10025 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tiny llama sample'}
+2024-08-04 21:19:47,164 INFO    MainThread:10025 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-08-04 21:19:47,164 INFO    MainThread:10025 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'}
+2024-08-04 21:19:47,164 INFO    MainThread:10025 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240804_211947-niq3ake5/logs/debug.log
+2024-08-04 21:19:47,164 INFO    MainThread:10025 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240804_211947-niq3ake5/logs/debug-internal.log
+2024-08-04 21:19:47,164 INFO    MainThread:10025 [wandb_init.py:init():566] calling init triggers
+2024-08-04 21:19:47,164 INFO    MainThread:10025 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
+config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 512, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-llama_train_2024-08-04-21:19:16', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-llama', 'save': '/work/llm_recipes/models/tiny-llama', 'base_model': '/share/pretrained_lm/meta-llama/TinyLlama_v1.1', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 2000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 2000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 8, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-llama', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32000, 'gradient_accumulation_steps': 40}
+2024-08-04 21:19:47,164 INFO    MainThread:10025 [wandb_init.py:init():616] starting backend
+2024-08-04 21:19:47,164 INFO    MainThread:10025 [wandb_init.py:init():620] setting up manager
+2024-08-04 21:19:47,169 INFO    MainThread:10025 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-08-04 21:19:47,170 INFO    MainThread:10025 [wandb_init.py:init():628] backend started and connected
+2024-08-04 21:19:47,175 INFO    MainThread:10025 [wandb_init.py:init():720] updated telemetry
+2024-08-04 21:19:47,312 INFO    MainThread:10025 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-08-04 21:19:47,827 INFO    MainThread:10025 [wandb_run.py:_on_init():2262] communicating current version
+2024-08-04 21:19:47,910 INFO    MainThread:10025 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-08-04 21:19:47,910 INFO    MainThread:10025 [wandb_init.py:init():804] starting run threads in backend
+2024-08-04 21:19:47,990 INFO    MainThread:10025 [wandb_run.py:_console_start():2241] atexit reg
+2024-08-04 21:19:47,991 INFO    MainThread:10025 [wandb_run.py:_redirect():2096] redirect: wrap_raw
+2024-08-04 21:19:47,991 INFO    MainThread:10025 [wandb_run.py:_redirect():2161] Wrapping output streams.
+2024-08-04 21:19:47,991 INFO    MainThread:10025 [wandb_run.py:_redirect():2186] Redirects installed.
+2024-08-04 21:19:47,992 INFO    MainThread:10025 [wandb_init.py:init():847] run started, returning control to user process
+2024-08-04 21:20:54,436 INFO    MainThread:10025 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'silu', 'hidden_size': 2048, 'model_type': 'llama', 'max_position_embeddings': 2048, 'num_attention_heads': 32, 'num_hidden_layers': 22, 'model_architecture': 'LlamaForCausalLM'}
+2024-08-04 21:20:54,436 INFO    MainThread:10025 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1}
+2024-08-04 21:21:00,034 WARNING MsgRouterThr:10025 [router.py:message_loop():77] message_loop has been closed

wandb/run-20240812_055620-qpw0uqx2/files/config.yaml

@@ -0,0 +1,314 @@
+wandb_version: 1
+
+sharding_strategy:
+  desc: null
+  value: FULL_SHARD
+checkpoint_type:
+  desc: null
+  value: LOCAL_STATE_DICT
+fsdp_activation_checkpointing:
+  desc: null
+  value: true
+fsdp_cpu_offload:
+  desc: null
+  value: false
+low_cpu_fsdp:
+  desc: null
+  value: false
+no_meta_device:
+  desc: null
+  value: false
+data_path:
+  desc: null
+  value: null
+split:
+  desc: null
+  value: 969, 30, 1
+train_data_path:
+  desc: null
+  value:
+  - '235289369'
+  - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document
+valid_data_path:
+  desc: null
+  value:
+  - '235289369'
+  - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document
+test_data_path:
+  desc: null
+  value:
+  - '235289369'
+  - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document
+data_cache_path:
+  desc: null
+  value: null
+vocab_size:
+  desc: null
+  value: null
+vocab_file:
+  desc: null
+  value: null
+merge_file:
+  desc: null
+  value: null
+seq_length:
+  desc: null
+  value: 4096
+num_workers:
+  desc: null
+  value: 2
+tokenizer_type:
+  desc: null
+  value: HFPreTrainedTokenizer
+tokenizer_model:
+  desc: null
+  value: /share/pretrained_lm/Phi/Phi-2
+reset_position_ids:
+  desc: null
+  value: false
+reset_attention_mask:
+  desc: null
+  value: false
+eod_mask_loss:
+  desc: null
+  value: false
+retro_return_doc_ids:
+  desc: null
+  value: false
+short_seq_prob:
+  desc: null
+  value: 0.1
+vocab_extra_ids:
+  desc: null
+  value: 0
+seed:
+  desc: null
+  value: 1234
+use_mpi:
+  desc: null
+  value: false
+wandb_entity:
+  desc: null
+  value: iwakawa-koichi-q5-tohoku-nlp6723
+wandb_name:
+  desc: null
+  value: yans-sample-Phi-2_train_2024-08-12-05:56:09
+wandb_project:
+  desc: null
+  value: llm_tutorial
+quantization:
+  desc: null
+  value: false
+use_freeze_layers:
+  desc: null
+  value: false
+freeze_layers:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+mixed_precision:
+  desc: null
+  value: true
+param_dtype:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: /work/llm_recipes/models/yans-sample-Phi-2
+save:
+  desc: null
+  value: /work/llm_recipes/models/yans-sample-Phi-2
+base_model:
+  desc: null
+  value: /share/pretrained_lm/Phi/Phi-2
+use_better_transformer:
+  desc: null
+  value: false
+grad_clip_norm:
+  desc: null
+  value: 1.0
+eval_interval:
+  desc: null
+  value: 200
+save_interval:
+  desc: null
+  value: 200
+eval_iters:
+  desc: null
+  value: 10
+optimizer:
+  desc: null
+  value: anyprecision
+lr:
+  desc: null
+  value: 2.0e-05
+lr_decay_style:
+  desc: null
+  value: cosine
+lr_decay_iters:
+  desc: null
+  value: 20000
+lr_warmup_iters:
+  desc: null
+  value: 500
+min_lr:
+  desc: null
+  value: 1.0e-06
+train_iters:
+  desc: null
+  value: 20000
+train_samples:
+  desc: null
+  value: null
+global_batch_size:
+  desc: null
+  value: 320
+micro_batch_size:
+  desc: null
+  value: 1
+make_vocab_size_divisible_by:
+  desc: null
+  value: 128
+sliding_window_size:
+  desc: null
+  value: 4096
+skip_batch:
+  desc: null
+  value: null
+no_save_optimizer_state:
+  desc: null
+  value: false
+continual_pretraining:
+  desc: null
+  value: false
+instruction_tuning:
+  desc: null
+  value: false
+direct_preference_optimization:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.1
+hidden_dropout:
+  desc: null
+  value: 0.1
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_eps:
+  desc: null
+  value: 1.0e-06
+hf_transformer_model_dir:
+  desc: null
+  value: null
+instruction_train_data_path:
+  desc: null
+  value: null
+instruction_valid_data_path:
+  desc: null
+  value: null
+epoch:
+  desc: null
+  value: null
+instruction_dataset_size:
+  desc: null
+  value: null
+save_sampler_state:
+  desc: null
+  value: false
+label_smoothing:
+  desc: null
+  value: 0.0
+save_n_checkpoints:
+  desc: null
+  value: 10
+hf_repo_id:
+  desc: null
+  value: koichi12/yans-sample-Phi-2
+create_public_hf_repo:
+  desc: null
+  value: false
+upload_all_checkpoints_to_hf:
+  desc: null
+  value: false
+hf_upload_retry_limit:
+  desc: null
+  value: 2
+exit_duration_in_mins:
+  desc: null
+  value: null
+source_key:
+  desc: null
+  value: null
+target_key:
+  desc: null
+  value: null
+attn_implementation:
+  desc: null
+  value: flash_attention_2
+efficient_instruction_tuning:
+  desc: null
+  value: false
+remove_padding_masking:
+  desc: null
+  value: false
+save_start_iter:
+  desc: null
+  value: null
+rank:
+  desc: null
+  value: 0
+world_size:
+  desc: null
+  value: 1
+padded_vocab_size:
+  desc: null
+  value: 50304
+gradient_accumulation_steps:
+  desc: null
+  value: 320
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.43.3
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1723409780.063771
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      2:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.43.3
+      8:
+      - 5
+      13: linux-x86_64

wandb/run-20240812_055620-qpw0uqx2/files/output.log

@@ -0,0 +1,9 @@
+Created Hugging Face repository with ID koichi12/yans-sample-Phi-2.
+Clearing GPU cache for all ranks
+--> Running with torch torch_distributed debug set to detail
+File not found: /work/llm_recipes/models/yans-sample-Phi-2/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/yans-sample-Phi-2/latest_iteration.txt
+File not found: /work/llm_recipes/models/yans-sample-Phi-2/latest_iteration.txt
+Unable to read latest iteration from /work/llm_recipes/models/yans-sample-Phi-2/latest_iteration.txt
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]

wandb/run-20240812_055620-qpw0uqx2/files/requirements.txt

@@ -0,0 +1,271 @@
+absl-py==2.1.0
+accelerate==0.33.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+apex==0.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+beautifulsoup4==4.12.3
+bleach==6.1.0
+blis==0.7.11
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+cmake==3.28.1
+colorama==0.4.6
+comm==0.2.1
+confection==0.1.4
+contourpy==1.2.0
+cubinlinker==0.3.0+2.g405ac64
+cuda-python==12.3.0rc4+9.gdb8c48a.dirty
+cudf==23.12.0
+cugraph-dgl==23.12.0
+cugraph-service-client==23.12.0
+cugraph-service-server==23.12.0
+cugraph==23.12.0
+cuml==23.12.0
+cupy-cuda12x==12.3.0
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.8
+dask-cuda==23.12.0
+dask-cudf==23.12.0
+dask==2023.11.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+distributed==2023.11.0
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+einops==0.7.0
+exceptiongroup==1.2.0
+execnet==2.0.2
+executing==2.0.1
+expecttest==0.1.3
+fastjsonschema==2.19.1
+fastrlock==0.8.2
+filelock==3.13.1
+flash-attn==2.4.2
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2023.12.2
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.27.0
+graphsurgeon==0.4.6
+grpcio==1.60.1
+huggingface-hub==0.24.5
+hypothesis==5.35.1
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+intel-openmp==2021.4.0
+ipadic==1.0.0
+ipykernel==6.29.2
+ipython-genutils==0.2.0
+ipython==8.21.0
+jedi==0.19.1
+jinja2==3.1.3
+joblib==1.3.2
+json5==0.9.14
+jsonnet==0.19.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.21.1
+jupyter-client==8.6.0
+jupyter-core==5.7.1
+jupyter-tensorboard==0.2.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==1.2.0
+jupyterlab==2.3.2
+jupytext==1.16.1
+kiwisolver==1.4.5
+langcodes==3.3.0
+lazy-loader==0.3
+librosa==0.10.1
+llvmlite==0.40.1
+locket==1.0.0
+logzero==1.7.0
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown==3.5.2
+markupsafe==2.1.4
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mecab-python3==1.0.6
+mistune==3.0.2
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mkl==2021.1.1
+mock==5.1.0
+more-itertools==9.1.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+murmurhash==1.0.10
+nbclient==0.9.0
+nbconvert==7.16.0
+nbformat==5.9.2
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.4.10
+numba==0.57.1+1.g1ff679645
+numpy==1.24.4
+nvfuser==0.1.4a0+d0bb811
+nvidia-dali-cuda120==1.34.0
+nvidia-pyindex==1.0.9
+nvtx==0.2.5
+oauthlib==3.2.2
+onnx==1.15.0rc2
+opencv==4.7.0
+optree==0.10.0
+packaging==23.2
+pandas==1.5.3
+pandocfilters==1.5.1
+parso==0.8.3
+partd==1.4.1
+peft==0.11.1
+pexpect==4.9.0
+pillow==10.2.0
+pip==24.0
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+polygraphy==0.49.4
+pooch==1.8.0
+portalocker==2.10.1
+preshed==3.0.9
+prettytable==3.9.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.24.4
+psutil==5.9.4
+ptxcompiler==0.8.1+2.g0d406d6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==14.0.1.dev0+gba5374836.d20240125
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pybind11-global==2.11.1
+pybind11==2.11.1
+pycocotools==2.0+nv0.8.0
+pycparser==2.21
+pydantic-core==2.16.2
+pydantic==2.6.1
+pygments==2.17.2
+pylibcugraph==23.12.0
+pylibcugraphops==23.12.0
+pylibraft==23.12.0
+pynvml==11.4.1
+pyparsing==3.1.1
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==13.0
+pytest-shard==0.1.2
+pytest-xdist==3.5.0
+pytest==8.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-hostlist==1.23.0
+pytorch-quantization==2.1.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.2
+raft-dask==23.12.0
+rapids-dask-dependency==23.12.1
+referencing==0.33.0
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests==2.31.0
+rich==13.7.0
+rmm==23.12.0
+rpds-py==0.17.1
+rsa==4.9
+sacrebleu==2.4.0
+safetensors==0.4.3
+scikit-learn==1.2.0
+scipy==1.12.0
+send2trash==1.8.2
+sentencepiece==0.1.99
+sentry-sdk==2.12.0
+setproctitle==1.3.3
+setuptools==68.2.2
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sphinx-glpi-theme==0.6
+srsly==2.4.8
+stack-data==0.6.3
+sympy==1.12
+tabulate==0.9.0
+tbb==2021.11.0
+tblib==3.0.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.9.0
+tensorrt==8.6.3
+terminado==0.18.0
+termplotlib==0.3.9
+thinc==8.2.3
+threadpoolctl==3.2.0
+thriftpy2==0.4.17
+tinycss2==1.2.1
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.1
+torch-tensorrt==2.3.0a0
+torch==2.3.0a0+ebedce2
+torchdata==0.7.1a0
+torchtext==0.17.0a0
+torchvision==0.18.0a0
+tornado==6.4
+tqdm==4.66.1
+traitlets==5.9.0
+transformer-engine==1.3.0+5b90b7f
+transformers==4.43.3
+treelite-runtime==3.9.1
+treelite==3.9.1
+triton==2.2.0+e28a256
+typer==0.9.0
+types-dataclasses==0.6.6
+typing-extensions==4.9.0
+ucx-py==0.35.0
+uff==0.6.9
+ujson==5.8.0
+urllib3==1.26.18
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.3.4
+webencodings==0.5.1
+werkzeug==3.0.1
+wheel==0.42.0
+xdoctest==1.0.2
+xgboost==1.7.6
+yarl==1.9.4
+zict==3.0.0
+zipp==3.17.0

wandb/run-20240812_055620-qpw0uqx2/files/wandb-metadata.json

@@ -0,0 +1,215 @@
+{
+    "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-08-11T20:56:20.724831",
+    "startedAt": "2024-08-11T20:56:20.050826",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--seq-length",
+        "4096",
+        "--sliding-window-size",
+        "4096",
+        "--micro-batch-size",
+        "1",
+        "--global-batch-size",
+        "320",
+        "--train-iters",
+        "20000",
+        "--tokenizer-type",
+        "HFPreTrainedTokenizer",
+        "--tokenizer-model",
+        "/share/pretrained_lm/Phi/Phi-2",
+        "--train-data-path",
+        "235289369",
+        "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document",
+        "--valid-data-path",
+        "235289369",
+        "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document",
+        "--test-data-path",
+        "235289369",
+        "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document",
+        "--lr",
+        "2e-5",
+        "--min-lr",
+        "1e-6",
+        "--lr-decay-style",
+        "cosine",
+        "--lr-warmup-iters",
+        "500",
+        "--lr-decay-iters",
+        "20000",
+        "--weight-decay",
+        "0.1",
+        "--grad-clip-norm",
+        "1.0",
+        "--optimizer",
+        "anyprecision",
+        "--adam-beta1",
+        "0.9",
+        "--adam-beta2",
+        "0.95",
+        "--adam-eps",
+        "1e-6",
+        "--save-interval",
+        "200",
+        "--eval-interval",
+        "200",
+        "--eval-iters",
+        "10",
+        "--bf16",
+        "--mixed-precision",
+        "--base-model",
+        "/share/pretrained_lm/Phi/Phi-2",
+        "--save",
+        "/work/llm_recipes/models/yans-sample-Phi-2",
+        "--load",
+        "/work/llm_recipes/models/yans-sample-Phi-2",
+        "--fsdp-activation-checkpointing",
+        "--sharding-strategy",
+        "FULL_SHARD",
+        "--checkpoint-type",
+        "LOCAL_STATE_DICT",
+        "--save-n-checkpoints",
+        "10",
+        "--hf-upload-retry-limit",
+        "2",
+        "--hf-repo-id",
+        "koichi12/yans-sample-Phi-2",
+        "--wandb-entity",
+        "iwakawa-koichi-q5-tohoku-nlp6723",
+        "--wandb-project",
+        "llm_tutorial",
+        "--wandb-name",
+        "yans-sample-Phi-2_train_2024-08-12-05:56:09"
+    ],
+    "state": "running",
+    "program": "/project/examples/finetuning.py",
+    "codePathLocal": "examples/finetuning.py",
+    "codePath": "examples/finetuning.py",
+    "git": {
+        "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git",
+        "commit": "6da01327e78c302bc0cfdb335f3ca297e2a19c8c"
+    },
+    "email": null,
+    "root": "/project",
+    "host": "gpu-koiwa-00",
+    "username": "koiwa",
+    "executable": "/usr/bin/python",
+    "cpu_count": 18,
+    "cpu_count_logical": 18,
+    "cpu_freq": {
+        "current": 2400.0429999999997,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 0.0625,
+            "used": 1.1444091796875e-05
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 56.487823486328125
+    }
+}

wandb/run-20240812_055620-qpw0uqx2/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 19}}

wandb/run-20240812_055620-qpw0uqx2/logs/debug-internal.log

@@ -0,0 +1,181 @@
+2024-08-12 05:56:20,065 INFO    StreamThr :11662 [internal.py:wandb_internal():86] W&B internal server running at pid: 11662, started at: 2024-08-12 05:56:20.064563
+2024-08-12 05:56:20,067 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: status
+2024-08-12 05:56:20,069 INFO    WriterThread:11662 [datastore.py:open_for_write():87] open: /project/wandb/run-20240812_055620-qpw0uqx2/run-qpw0uqx2.wandb
+2024-08-12 05:56:20,070 DEBUG   SenderThread:11662 [sender.py:send():382] send: header
+2024-08-12 05:56:20,085 DEBUG   SenderThread:11662 [sender.py:send():382] send: run
+2024-08-12 05:56:20,612 INFO    SenderThread:11662 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240812_055620-qpw0uqx2/files
+2024-08-12 05:56:20,612 INFO    SenderThread:11662 [sender.py:_start_run_threads():1136] run started: qpw0uqx2 with start time 1723409780.063771
+2024-08-12 05:56:20,617 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: check_version
+2024-08-12 05:56:20,617 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: check_version
+2024-08-12 05:56:20,704 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: run_start
+2024-08-12 05:56:20,711 DEBUG   HandlerThread:11662 [system_info.py:__init__():27] System info init
+2024-08-12 05:56:20,711 DEBUG   HandlerThread:11662 [system_info.py:__init__():42] System info init done
+2024-08-12 05:56:20,711 INFO    HandlerThread:11662 [system_monitor.py:start():194] Starting system monitor
+2024-08-12 05:56:20,711 INFO    SystemMonitor:11662 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-08-12 05:56:20,711 INFO    HandlerThread:11662 [system_monitor.py:probe():214] Collecting system info
+2024-08-12 05:56:20,712 INFO    SystemMonitor:11662 [interfaces.py:start():190] Started cpu monitoring
+2024-08-12 05:56:20,712 INFO    SystemMonitor:11662 [interfaces.py:start():190] Started disk monitoring
+2024-08-12 05:56:20,713 INFO    SystemMonitor:11662 [interfaces.py:start():190] Started gpu monitoring
+2024-08-12 05:56:20,714 INFO    SystemMonitor:11662 [interfaces.py:start():190] Started memory monitoring
+2024-08-12 05:56:20,714 INFO    SystemMonitor:11662 [interfaces.py:start():190] Started network monitoring
+2024-08-12 05:56:20,724 DEBUG   HandlerThread:11662 [system_info.py:probe():151] Probing system
+2024-08-12 05:56:20,729 DEBUG   HandlerThread:11662 [system_info.py:_probe_git():136] Probing git
+2024-08-12 05:56:20,742 DEBUG   HandlerThread:11662 [system_info.py:_probe_git():144] Probing git done
+2024-08-12 05:56:20,742 DEBUG   HandlerThread:11662 [system_info.py:probe():199] Probing system done
+2024-08-12 05:56:20,742 DEBUG   HandlerThread:11662 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-11T20:56:20.724831', 'startedAt': '2024-08-11T20:56:20.050826', 'docker': None, 'cuda': None, 'args': ('--seq-length', '4096', '--sliding-window-size', '4096', '--micro-batch-size', '1', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Phi/Phi-2', '--train-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--valid-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--test-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Phi/Phi-2', '--save', '/work/llm_recipes/models/yans-sample-Phi-2', '--load', '/work/llm_recipes/models/yans-sample-Phi-2', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-sample-Phi-2', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'yans-sample-Phi-2_train_2024-08-12-05:56:09'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '6da01327e78c302bc0cfdb335f3ca297e2a19c8c'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0429999999997, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.487823486328125}}
+2024-08-12 05:56:20,742 INFO    HandlerThread:11662 [system_monitor.py:probe():224] Finished collecting system info
+2024-08-12 05:56:20,742 INFO    HandlerThread:11662 [system_monitor.py:probe():227] Publishing system info
+2024-08-12 05:56:20,743 INFO    HandlerThread:11662 [system_monitor.py:probe():229] Finished publishing system info
+2024-08-12 05:56:20,749 DEBUG   SenderThread:11662 [sender.py:send():382] send: files
+2024-08-12 05:56:20,749 INFO    SenderThread:11662 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now
+2024-08-12 05:56:20,759 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: python_packages
+2024-08-12 05:56:20,759 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-12 05:56:20,759 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: python_packages
+2024-08-12 05:56:20,760 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 05:56:20,761 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: stop_status
+2024-08-12 05:56:21,039 DEBUG   SenderThread:11662 [sender.py:send():382] send: telemetry
+2024-08-12 05:56:21,402 INFO    wandb-upload_0:11662 [upload_job.py:push():131] Uploaded file /tmp/tmp1ghcluufwandb/07mrguha-wandb-metadata.json
+2024-08-12 05:56:21,614 INFO    Thread-12 :11662 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_055620-qpw0uqx2/files/output.log
+2024-08-12 05:56:21,614 INFO    Thread-12 :11662 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_055620-qpw0uqx2/files/requirements.txt
+2024-08-12 05:56:21,614 INFO    Thread-12 :11662 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_055620-qpw0uqx2/files/wandb-metadata.json
+2024-08-12 05:56:23,614 INFO    Thread-12 :11662 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_055620-qpw0uqx2/files/output.log
+2024-08-12 05:56:25,807 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 05:56:30,807 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 05:56:35,758 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-12 05:56:35,759 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-12 05:56:35,759 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: stop_status
+2024-08-12 05:56:35,962 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 05:56:40,657 DEBUG   SenderThread:11662 [sender.py:send():382] send: exit
+2024-08-12 05:56:40,657 INFO    SenderThread:11662 [sender.py:send_exit():589] handling exit code: 255
+2024-08-12 05:56:40,657 INFO    SenderThread:11662 [sender.py:send_exit():591] handling runtime: 19
+2024-08-12 05:56:40,659 INFO    SenderThread:11662 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-12 05:56:40,659 INFO    SenderThread:11662 [sender.py:send_exit():597] send defer
+2024-08-12 05:56:40,659 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:40,659 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 0
+2024-08-12 05:56:40,659 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:40,659 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 0
+2024-08-12 05:56:40,659 INFO    SenderThread:11662 [sender.py:transition_state():617] send defer: 1
+2024-08-12 05:56:40,659 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:40,660 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 1
+2024-08-12 05:56:40,660 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:40,660 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 1
+2024-08-12 05:56:40,660 INFO    SenderThread:11662 [sender.py:transition_state():617] send defer: 2
+2024-08-12 05:56:40,660 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:40,660 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 2
+2024-08-12 05:56:40,660 INFO    HandlerThread:11662 [system_monitor.py:finish():203] Stopping system monitor
+2024-08-12 05:56:40,660 DEBUG   SystemMonitor:11662 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-08-12 05:56:40,660 INFO    HandlerThread:11662 [interfaces.py:finish():202] Joined cpu monitor
+2024-08-12 05:56:40,660 DEBUG   SystemMonitor:11662 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-08-12 05:56:40,661 INFO    HandlerThread:11662 [interfaces.py:finish():202] Joined disk monitor
+2024-08-12 05:56:40,661 DEBUG   SystemMonitor:11662 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-08-12 05:56:40,693 INFO    HandlerThread:11662 [interfaces.py:finish():202] Joined gpu monitor
+2024-08-12 05:56:40,693 INFO    HandlerThread:11662 [interfaces.py:finish():202] Joined memory monitor
+2024-08-12 05:56:40,693 INFO    HandlerThread:11662 [interfaces.py:finish():202] Joined network monitor
+2024-08-12 05:56:40,693 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:40,694 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 2
+2024-08-12 05:56:40,694 INFO    SenderThread:11662 [sender.py:transition_state():617] send defer: 3
+2024-08-12 05:56:40,694 DEBUG   SenderThread:11662 [sender.py:send():382] send: stats
+2024-08-12 05:56:40,694 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:40,694 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 3
+2024-08-12 05:56:40,694 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:40,694 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 3
+2024-08-12 05:56:40,694 INFO    SenderThread:11662 [sender.py:transition_state():617] send defer: 4
+2024-08-12 05:56:40,694 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:40,694 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 4
+2024-08-12 05:56:40,694 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:40,694 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 4
+2024-08-12 05:56:40,695 INFO    SenderThread:11662 [sender.py:transition_state():617] send defer: 5
+2024-08-12 05:56:40,695 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:40,695 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 5
+2024-08-12 05:56:40,695 DEBUG   SenderThread:11662 [sender.py:send():382] send: summary
+2024-08-12 05:56:40,696 INFO    SenderThread:11662 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-12 05:56:40,696 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:40,696 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 5
+2024-08-12 05:56:40,696 INFO    SenderThread:11662 [sender.py:transition_state():617] send defer: 6
+2024-08-12 05:56:40,696 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:40,696 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 6
+2024-08-12 05:56:40,696 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:40,696 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 6
+2024-08-12 05:56:40,699 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 05:56:40,927 INFO    SenderThread:11662 [sender.py:transition_state():617] send defer: 7
+2024-08-12 05:56:40,927 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:40,928 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 7
+2024-08-12 05:56:40,928 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:40,928 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 7
+2024-08-12 05:56:41,625 INFO    Thread-12 :11662 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_055620-qpw0uqx2/files/config.yaml
+2024-08-12 05:56:41,626 INFO    Thread-12 :11662 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_055620-qpw0uqx2/files/wandb-summary.json
+2024-08-12 05:56:41,657 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-12 05:56:41,724 INFO    SenderThread:11662 [sender.py:transition_state():617] send defer: 8
+2024-08-12 05:56:41,724 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: poll_exit
+2024-08-12 05:56:41,724 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:41,724 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 8
+2024-08-12 05:56:41,724 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:41,725 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 8
+2024-08-12 05:56:41,725 INFO    SenderThread:11662 [job_builder.py:build():296] Attempting to build job artifact
+2024-08-12 05:56:41,725 INFO    SenderThread:11662 [job_builder.py:_get_source_type():426] is repo sourced job
+2024-08-12 05:56:41,740 INFO    SenderThread:11662 [job_builder.py:build():402] adding wandb-job metadata file
+2024-08-12 05:56:41,748 INFO    SenderThread:11662 [sender.py:transition_state():617] send defer: 9
+2024-08-12 05:56:41,749 DEBUG   SenderThread:11662 [sender.py:send():382] send: artifact
+2024-08-12 05:56:41,749 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:41,750 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 9
+2024-08-12 05:56:42,618 INFO    SenderThread:11662 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTEzOTgzMzc4Mw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTEzOTg5OTc5MQ==', 'versionIndex': 7}}}
+2024-08-12 05:56:42,618 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:42,618 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 9
+2024-08-12 05:56:42,618 INFO    SenderThread:11662 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-08-12 05:56:42,626 INFO    Thread-12 :11662 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_055620-qpw0uqx2/files/output.log
+2024-08-12 05:56:42,627 INFO    SenderThread:11662 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240812_055620-qpw0uqx2/files
+2024-08-12 05:56:42,627 INFO    SenderThread:11662 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_055620-qpw0uqx2/files/requirements.txt requirements.txt
+2024-08-12 05:56:42,627 INFO    SenderThread:11662 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_055620-qpw0uqx2/files/config.yaml config.yaml
+2024-08-12 05:56:42,629 INFO    SenderThread:11662 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_055620-qpw0uqx2/files/wandb-metadata.json wandb-metadata.json
+2024-08-12 05:56:42,630 INFO    SenderThread:11662 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_055620-qpw0uqx2/files/wandb-summary.json wandb-summary.json
+2024-08-12 05:56:42,631 INFO    SenderThread:11662 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_055620-qpw0uqx2/files/output.log output.log
+2024-08-12 05:56:42,631 INFO    SenderThread:11662 [sender.py:transition_state():617] send defer: 10
+2024-08-12 05:56:42,633 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:42,633 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 10
+2024-08-12 05:56:42,633 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:42,634 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 10
+2024-08-12 05:56:42,635 INFO    SenderThread:11662 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-12 05:56:42,657 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-12 05:56:42,657 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: poll_exit
+2024-08-12 05:56:43,034 INFO    wandb-upload_0:11662 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_055620-qpw0uqx2/files/requirements.txt
+2024-08-12 05:56:43,159 INFO    wandb-upload_1:11662 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_055620-qpw0uqx2/files/config.yaml
+2024-08-12 05:56:43,201 INFO    wandb-upload_2:11662 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_055620-qpw0uqx2/files/wandb-summary.json
+2024-08-12 05:56:43,202 INFO    wandb-upload_3:11662 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_055620-qpw0uqx2/files/output.log
+2024-08-12 05:56:43,402 INFO    Thread-11 (_thread_body):11662 [sender.py:transition_state():617] send defer: 11
+2024-08-12 05:56:43,403 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:43,403 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 11
+2024-08-12 05:56:43,403 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:43,403 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 11
+2024-08-12 05:56:43,403 INFO    SenderThread:11662 [file_pusher.py:join():178] waiting for file pusher
+2024-08-12 05:56:43,403 INFO    SenderThread:11662 [sender.py:transition_state():617] send defer: 12
+2024-08-12 05:56:43,403 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:43,404 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 12
+2024-08-12 05:56:43,404 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:43,404 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 12
+2024-08-12 05:56:43,404 INFO    SenderThread:11662 [file_stream.py:finish():595] file stream finish called
+2024-08-12 05:56:43,591 INFO    SenderThread:11662 [file_stream.py:finish():599] file stream finish is done
+2024-08-12 05:56:43,591 INFO    SenderThread:11662 [sender.py:transition_state():617] send defer: 13
+2024-08-12 05:56:43,591 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:43,591 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 13
+2024-08-12 05:56:43,592 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:43,592 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 13
+2024-08-12 05:56:43,592 INFO    SenderThread:11662 [sender.py:transition_state():617] send defer: 14
+2024-08-12 05:56:43,592 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: defer
+2024-08-12 05:56:43,592 DEBUG   SenderThread:11662 [sender.py:send():382] send: final
+2024-08-12 05:56:43,592 INFO    HandlerThread:11662 [handler.py:handle_request_defer():172] handle defer: 14
+2024-08-12 05:56:43,592 DEBUG   SenderThread:11662 [sender.py:send():382] send: footer
+2024-08-12 05:56:43,593 DEBUG   SenderThread:11662 [sender.py:send_request():409] send_request: defer
+2024-08-12 05:56:43,593 INFO    SenderThread:11662 [sender.py:send_request_defer():613] handle sender defer: 14
+2024-08-12 05:56:47,593 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 05:56:52,594 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 05:56:57,595 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 05:57:02,595 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 05:57:07,596 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 05:57:12,597 DEBUG   HandlerThread:11662 [handler.py:handle_request():146] handle_request: status_report
+2024-08-12 05:57:17,123 WARNING StreamThr :11662 [internal.py:is_dead():414] Internal process exiting, parent pid 11591 disappeared
+2024-08-12 05:57:17,123 ERROR   StreamThr :11662 [internal.py:wandb_internal():152] Internal process shutdown.
+2024-08-12 05:57:17,597 INFO    SenderThread:11662 [sender.py:finish():1572] shutting down sender
+2024-08-12 05:57:17,597 INFO    SenderThread:11662 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-12 05:57:17,597 INFO    SenderThread:11662 [file_pusher.py:join():178] waiting for file pusher
+2024-08-12 05:57:17,598 INFO    WriterThread:11662 [datastore.py:close():296] close: /project/wandb/run-20240812_055620-qpw0uqx2/run-qpw0uqx2.wandb
+2024-08-12 05:57:17,598 INFO    HandlerThread:11662 [handler.py:finish():869] shutting down handler

wandb/run-20240812_055620-qpw0uqx2/logs/debug.log

@@ -0,0 +1,27 @@
+2024-08-12 05:56:20,056 INFO    MainThread:11591 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
+2024-08-12 05:56:20,057 INFO    MainThread:11591 [wandb_setup.py:_flush():76] Configure stats pid to 11591
+2024-08-12 05:56:20,057 INFO    MainThread:11591 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings
+2024-08-12 05:56:20,057 INFO    MainThread:11591 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings
+2024-08-12 05:56:20,057 INFO    MainThread:11591 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train phi'}
+2024-08-12 05:56:20,057 INFO    MainThread:11591 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-08-12 05:56:20,057 INFO    MainThread:11591 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'}
+2024-08-12 05:56:20,057 INFO    MainThread:11591 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240812_055620-qpw0uqx2/logs/debug.log
+2024-08-12 05:56:20,057 INFO    MainThread:11591 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240812_055620-qpw0uqx2/logs/debug-internal.log
+2024-08-12 05:56:20,057 INFO    MainThread:11591 [wandb_init.py:init():566] calling init triggers
+2024-08-12 05:56:20,057 INFO    MainThread:11591 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
+config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'valid_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'test_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 4096, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Phi/Phi-2', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-sample-Phi-2_train_2024-08-12-05:56:09', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-sample-Phi-2', 'save': '/work/llm_recipes/models/yans-sample-Phi-2', 'base_model': '/share/pretrained_lm/Phi/Phi-2', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 1, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-sample-Phi-2', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 50304, 'gradient_accumulation_steps': 320}
+2024-08-12 05:56:20,057 INFO    MainThread:11591 [wandb_init.py:init():616] starting backend
+2024-08-12 05:56:20,057 INFO    MainThread:11591 [wandb_init.py:init():620] setting up manager
+2024-08-12 05:56:20,062 INFO    MainThread:11591 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-08-12 05:56:20,063 INFO    MainThread:11591 [wandb_init.py:init():628] backend started and connected
+2024-08-12 05:56:20,068 INFO    MainThread:11591 [wandb_init.py:init():720] updated telemetry
+2024-08-12 05:56:20,080 INFO    MainThread:11591 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-08-12 05:56:20,616 INFO    MainThread:11591 [wandb_run.py:_on_init():2262] communicating current version
+2024-08-12 05:56:20,697 INFO    MainThread:11591 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.6 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-08-12 05:56:20,697 INFO    MainThread:11591 [wandb_init.py:init():804] starting run threads in backend
+2024-08-12 05:56:20,758 INFO    MainThread:11591 [wandb_run.py:_console_start():2241] atexit reg
+2024-08-12 05:56:20,758 INFO    MainThread:11591 [wandb_run.py:_redirect():2096] redirect: wrap_raw
+2024-08-12 05:56:20,759 INFO    MainThread:11591 [wandb_run.py:_redirect():2161] Wrapping output streams.
+2024-08-12 05:56:20,759 INFO    MainThread:11591 [wandb_run.py:_redirect():2186] Redirects installed.
+2024-08-12 05:56:20,760 INFO    MainThread:11591 [wandb_init.py:init():847] run started, returning control to user process

wandb/run-20240812_073955-ikoro1zp/files/config.yaml

@@ -0,0 +1,335 @@
+wandb_version: 1
+
+sharding_strategy:
+  desc: null
+  value: FULL_SHARD
+checkpoint_type:
+  desc: null
+  value: LOCAL_STATE_DICT
+fsdp_activation_checkpointing:
+  desc: null
+  value: true
+fsdp_cpu_offload:
+  desc: null
+  value: false
+low_cpu_fsdp:
+  desc: null
+  value: false
+no_meta_device:
+  desc: null
+  value: false
+data_path:
+  desc: null
+  value: null
+split:
+  desc: null
+  value: 969, 30, 1
+train_data_path:
+  desc: null
+  value:
+  - '304771887'
+  - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document
+valid_data_path:
+  desc: null
+  value:
+  - '304771887'
+  - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document
+test_data_path:
+  desc: null
+  value:
+  - '304771887'
+  - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document
+data_cache_path:
+  desc: null
+  value: null
+vocab_size:
+  desc: null
+  value: null
+vocab_file:
+  desc: null
+  value: null
+merge_file:
+  desc: null
+  value: null
+seq_length:
+  desc: null
+  value: 4096
+num_workers:
+  desc: null
+  value: 2
+tokenizer_type:
+  desc: null
+  value: HFPreTrainedTokenizer
+tokenizer_model:
+  desc: null
+  value: /share/pretrained_lm/Qwen/Qwen2-0.5B
+reset_position_ids:
+  desc: null
+  value: false
+reset_attention_mask:
+  desc: null
+  value: false
+eod_mask_loss:
+  desc: null
+  value: false
+retro_return_doc_ids:
+  desc: null
+  value: false
+short_seq_prob:
+  desc: null
+  value: 0.1
+vocab_extra_ids:
+  desc: null
+  value: 0
+seed:
+  desc: null
+  value: 1234
+use_mpi:
+  desc: null
+  value: false
+wandb_entity:
+  desc: null
+  value: iwakawa-koichi-q5-tohoku-nlp6723
+wandb_name:
+  desc: null
+  value: yans-qwen2-0.5B_train_2024-08-12-07:39:43
+wandb_project:
+  desc: null
+  value: llm_tutorial
+quantization:
+  desc: null
+  value: false
+use_freeze_layers:
+  desc: null
+  value: false
+freeze_layers:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+mixed_precision:
+  desc: null
+  value: true
+param_dtype:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: /work/llm_recipes/models/yans-qwen2-0.5B
+save:
+  desc: null
+  value: /work/llm_recipes/models/yans-qwen2-0.5B
+base_model:
+  desc: null
+  value: /share/pretrained_lm/Qwen/Qwen2-0.5B
+use_better_transformer:
+  desc: null
+  value: false
+grad_clip_norm:
+  desc: null
+  value: 1.0
+eval_interval:
+  desc: null
+  value: 10
+save_interval:
+  desc: null
+  value: 10
+eval_iters:
+  desc: null
+  value: 10
+optimizer:
+  desc: null
+  value: adam
+lr:
+  desc: null
+  value: 2.0e-05
+lr_decay_style:
+  desc: null
+  value: cosine
+lr_decay_iters:
+  desc: null
+  value: 20000
+lr_warmup_iters:
+  desc: null
+  value: 500
+min_lr:
+  desc: null
+  value: 1.0e-06
+train_iters:
+  desc: null
+  value: 20000
+train_samples:
+  desc: null
+  value: null
+global_batch_size:
+  desc: null
+  value: 320
+micro_batch_size:
+  desc: null
+  value: 1
+make_vocab_size_divisible_by:
+  desc: null
+  value: 128
+sliding_window_size:
+  desc: null
+  value: 4096
+skip_batch:
+  desc: null
+  value: null
+no_save_optimizer_state:
+  desc: null
+  value: false
+continual_pretraining:
+  desc: null
+  value: false
+instruction_tuning:
+  desc: null
+  value: false
+direct_preference_optimization:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.1
+hidden_dropout:
+  desc: null
+  value: 0.1
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_eps:
+  desc: null
+  value: 1.0e-06
+hf_transformer_model_dir:
+  desc: null
+  value: null
+instruction_train_data_path:
+  desc: null
+  value: null
+instruction_valid_data_path:
+  desc: null
+  value: null
+epoch:
+  desc: null
+  value: null
+instruction_dataset_size:
+  desc: null
+  value: null
+save_sampler_state:
+  desc: null
+  value: false
+label_smoothing:
+  desc: null
+  value: 0.0
+save_n_checkpoints:
+  desc: null
+  value: 10
+hf_repo_id:
+  desc: null
+  value: koichi12/yans-qwen2-0.5B
+create_public_hf_repo:
+  desc: null
+  value: false
+upload_all_checkpoints_to_hf:
+  desc: null
+  value: false
+hf_upload_retry_limit:
+  desc: null
+  value: 2
+exit_duration_in_mins:
+  desc: null
+  value: null
+source_key:
+  desc: null
+  value: null
+target_key:
+  desc: null
+  value: null
+attn_implementation:
+  desc: null
+  value: flash_attention_2
+efficient_instruction_tuning:
+  desc: null
+  value: false
+remove_padding_masking:
+  desc: null
+  value: false
+save_start_iter:
+  desc: null
+  value: null
+rank:
+  desc: null
+  value: 0
+world_size:
+  desc: null
+  value: 1
+padded_vocab_size:
+  desc: null
+  value: 151680
+gradient_accumulation_steps:
+  desc: null
+  value: 320
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.43.3
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1723415995.685329
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      2:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.43.3
+      8:
+      - 5
+      13: linux-x86_64
+model_architecture:
+  desc: null
+  value: Qwen2ForCausalLM
+activation_function:
+  desc: null
+  value: silu
+hidden_size:
+  desc: null
+  value: 896
+model_type:
+  desc: null
+  value: qwen2
+max_position_embeddings:
+  desc: null
+  value: 4096
+num_attention_heads:
+  desc: null
+  value: 14
+num_hidden_layers:
+  desc: null
+  value: 24

wandb/run-20240812_073955-ikoro1zp/files/requirements.txt

@@ -0,0 +1,271 @@
+absl-py==2.1.0
+accelerate==0.33.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+apex==0.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+beautifulsoup4==4.12.3
+bleach==6.1.0
+blis==0.7.11
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+cmake==3.28.1
+colorama==0.4.6
+comm==0.2.1
+confection==0.1.4
+contourpy==1.2.0
+cubinlinker==0.3.0+2.g405ac64
+cuda-python==12.3.0rc4+9.gdb8c48a.dirty
+cudf==23.12.0
+cugraph-dgl==23.12.0
+cugraph-service-client==23.12.0
+cugraph-service-server==23.12.0
+cugraph==23.12.0
+cuml==23.12.0
+cupy-cuda12x==12.3.0
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.8
+dask-cuda==23.12.0
+dask-cudf==23.12.0
+dask==2023.11.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+distributed==2023.11.0
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+einops==0.7.0
+exceptiongroup==1.2.0
+execnet==2.0.2
+executing==2.0.1
+expecttest==0.1.3
+fastjsonschema==2.19.1
+fastrlock==0.8.2
+filelock==3.13.1
+flash-attn==2.4.2
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2023.12.2
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.27.0
+graphsurgeon==0.4.6
+grpcio==1.60.1
+huggingface-hub==0.24.5
+hypothesis==5.35.1
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+intel-openmp==2021.4.0
+ipadic==1.0.0
+ipykernel==6.29.2
+ipython-genutils==0.2.0
+ipython==8.21.0
+jedi==0.19.1
+jinja2==3.1.3
+joblib==1.3.2
+json5==0.9.14
+jsonnet==0.19.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.21.1
+jupyter-client==8.6.0
+jupyter-core==5.7.1
+jupyter-tensorboard==0.2.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==1.2.0
+jupyterlab==2.3.2
+jupytext==1.16.1
+kiwisolver==1.4.5
+langcodes==3.3.0
+lazy-loader==0.3
+librosa==0.10.1
+llvmlite==0.40.1
+locket==1.0.0
+logzero==1.7.0
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown==3.5.2
+markupsafe==2.1.4
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mecab-python3==1.0.6
+mistune==3.0.2
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mkl==2021.1.1
+mock==5.1.0
+more-itertools==9.1.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+murmurhash==1.0.10
+nbclient==0.9.0
+nbconvert==7.16.0
+nbformat==5.9.2
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.4.10
+numba==0.57.1+1.g1ff679645
+numpy==1.24.4
+nvfuser==0.1.4a0+d0bb811
+nvidia-dali-cuda120==1.34.0
+nvidia-pyindex==1.0.9
+nvtx==0.2.5
+oauthlib==3.2.2
+onnx==1.15.0rc2
+opencv==4.7.0
+optree==0.10.0
+packaging==23.2
+pandas==1.5.3
+pandocfilters==1.5.1
+parso==0.8.3
+partd==1.4.1
+peft==0.11.1
+pexpect==4.9.0
+pillow==10.2.0
+pip==24.0
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+polygraphy==0.49.4
+pooch==1.8.0
+portalocker==2.10.1
+preshed==3.0.9
+prettytable==3.9.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.24.4
+psutil==5.9.4
+ptxcompiler==0.8.1+2.g0d406d6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==14.0.1.dev0+gba5374836.d20240125
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pybind11-global==2.11.1
+pybind11==2.11.1
+pycocotools==2.0+nv0.8.0
+pycparser==2.21
+pydantic-core==2.16.2
+pydantic==2.6.1
+pygments==2.17.2
+pylibcugraph==23.12.0
+pylibcugraphops==23.12.0
+pylibraft==23.12.0
+pynvml==11.4.1
+pyparsing==3.1.1
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==13.0
+pytest-shard==0.1.2
+pytest-xdist==3.5.0
+pytest==8.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-hostlist==1.23.0
+pytorch-quantization==2.1.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.2
+raft-dask==23.12.0
+rapids-dask-dependency==23.12.1
+referencing==0.33.0
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests==2.31.0
+rich==13.7.0
+rmm==23.12.0
+rpds-py==0.17.1
+rsa==4.9
+sacrebleu==2.4.0
+safetensors==0.4.3
+scikit-learn==1.2.0
+scipy==1.12.0
+send2trash==1.8.2
+sentencepiece==0.1.99
+sentry-sdk==2.12.0
+setproctitle==1.3.3
+setuptools==68.2.2
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sphinx-glpi-theme==0.6
+srsly==2.4.8
+stack-data==0.6.3
+sympy==1.12
+tabulate==0.9.0
+tbb==2021.11.0
+tblib==3.0.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.9.0
+tensorrt==8.6.3
+terminado==0.18.0
+termplotlib==0.3.9
+thinc==8.2.3
+threadpoolctl==3.2.0
+thriftpy2==0.4.17
+tinycss2==1.2.1
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+toolz==0.12.1
+torch-tensorrt==2.3.0a0
+torch==2.3.0a0+ebedce2
+torchdata==0.7.1a0
+torchtext==0.17.0a0
+torchvision==0.18.0a0
+tornado==6.4
+tqdm==4.66.1
+traitlets==5.9.0
+transformer-engine==1.3.0+5b90b7f
+transformers==4.43.3
+treelite-runtime==3.9.1
+treelite==3.9.1
+triton==2.2.0+e28a256
+typer==0.9.0
+types-dataclasses==0.6.6
+typing-extensions==4.9.0
+ucx-py==0.35.0
+uff==0.6.9
+ujson==5.8.0
+urllib3==1.26.18
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.3.4
+webencodings==0.5.1
+werkzeug==3.0.1
+wheel==0.42.0
+xdoctest==1.0.2
+xgboost==1.7.6
+yarl==1.9.4
+zict==3.0.0
+zipp==3.17.0

wandb/run-20240812_073955-ikoro1zp/files/wandb-metadata.json

@@ -0,0 +1,215 @@
+{
+    "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-08-11T22:39:56.314869",
+    "startedAt": "2024-08-11T22:39:55.672249",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--seq-length",
+        "4096",
+        "--sliding-window-size",
+        "4096",
+        "--micro-batch-size",
+        "1",
+        "--global-batch-size",
+        "320",
+        "--train-iters",
+        "20000",
+        "--tokenizer-type",
+        "HFPreTrainedTokenizer",
+        "--tokenizer-model",
+        "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+        "--train-data-path",
+        "304771887",
+        "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document",
+        "--valid-data-path",
+        "304771887",
+        "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document",
+        "--test-data-path",
+        "304771887",
+        "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document",
+        "--lr",
+        "2e-5",
+        "--min-lr",
+        "1e-6",
+        "--lr-decay-style",
+        "cosine",
+        "--lr-warmup-iters",
+        "500",
+        "--lr-decay-iters",
+        "20000",
+        "--weight-decay",
+        "0.1",
+        "--grad-clip-norm",
+        "1.0",
+        "--optimizer",
+        "adam",
+        "--adam-beta1",
+        "0.9",
+        "--adam-beta2",
+        "0.95",
+        "--adam-eps",
+        "1e-6",
+        "--save-interval",
+        "10",
+        "--eval-interval",
+        "10",
+        "--eval-iters",
+        "10",
+        "--bf16",
+        "--mixed-precision",
+        "--base-model",
+        "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+        "--save",
+        "/work/llm_recipes/models/yans-qwen2-0.5B",
+        "--load",
+        "/work/llm_recipes/models/yans-qwen2-0.5B",
+        "--fsdp-activation-checkpointing",
+        "--sharding-strategy",
+        "FULL_SHARD",
+        "--checkpoint-type",
+        "LOCAL_STATE_DICT",
+        "--save-n-checkpoints",
+        "10",
+        "--hf-upload-retry-limit",
+        "2",
+        "--hf-repo-id",
+        "koichi12/yans-qwen2-0.5B",
+        "--wandb-entity",
+        "iwakawa-koichi-q5-tohoku-nlp6723",
+        "--wandb-project",
+        "llm_tutorial",
+        "--wandb-name",
+        "yans-qwen2-0.5B_train_2024-08-12-07:39:43"
+    ],
+    "state": "running",
+    "program": "/project/examples/finetuning.py",
+    "codePathLocal": "examples/finetuning.py",
+    "codePath": "examples/finetuning.py",
+    "git": {
+        "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git",
+        "commit": "6da01327e78c302bc0cfdb335f3ca297e2a19c8c"
+    },
+    "email": null,
+    "root": "/project",
+    "host": "gpu-koiwa-00",
+    "username": "koiwa",
+    "executable": "/usr/bin/python",
+    "cpu_count": 18,
+    "cpu_count_logical": 18,
+    "cpu_freq": {
+        "current": 2400.0429999999997,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.043,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 0.0625,
+            "used": 1.1444091796875e-05
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 56.487823486328125
+    }
+}

wandb/run-20240812_073955-ikoro1zp/files/wandb-summary.json

@@ -0,0 +1 @@
+{"training/loss": 2.4635202884674072, "training/perplexity": 11.746088463770842, "utils/batch_size": 1, "utils/global_batch_size": 320, "utils/seq_len": 4097, "utils/gradient_accumulation_steps": 320, "utils/iteration": 1167, "optimizer/lr": 1.9945203423500063e-05, "optimizer/variance_l2": 0.0048320659825907535, "optimizer/variance_sqrt_l2": 0.5318417899390797, "optimizer/momentum_l2": 0.127020084622386, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 0.2829437255859375, "optimizer/variance_sqrt_l1": 4615.0, "optimizer/momentum_l1": 979.125, "optimizer/weight_l1": 6918144.0, "optimizer/variance_abs_max": 0.0029296875, "optimizer/variance_sqrt_abs_max": 0.05419921875, "optimizer/momentum_abs_max": 0.01129150390625, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 73.68068221400608, "stats/tokens_per_sec": 17793.53774429062, "stats/tokens_per_sec_per_gpu": 17793.53774429062, "stats/tflops": 71.54763648032535, "_timestamp": 1723503194.8273196, "_runtime": 87199.14199066162, "_step": 1167, "evaluation/val_loss": 2.4397435188293457, "evaluation/val_ppl": 11.470099449157715, "_wandb": {"runtime": 87227}}

wandb/run-20240812_073955-ikoro1zp/logs/debug.log

@@ -0,0 +1,29 @@
+2024-08-12 07:39:55,678 INFO    MainThread:14724 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
+2024-08-12 07:39:55,678 INFO    MainThread:14724 [wandb_setup.py:_flush():76] Configure stats pid to 14724
+2024-08-12 07:39:55,678 INFO    MainThread:14724 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings
+2024-08-12 07:39:55,678 INFO    MainThread:14724 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings
+2024-08-12 07:39:55,678 INFO    MainThread:14724 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train Qwen2'}
+2024-08-12 07:39:55,679 INFO    MainThread:14724 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-08-12 07:39:55,679 INFO    MainThread:14724 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'}
+2024-08-12 07:39:55,679 INFO    MainThread:14724 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240812_073955-ikoro1zp/logs/debug.log
+2024-08-12 07:39:55,679 INFO    MainThread:14724 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240812_073955-ikoro1zp/logs/debug-internal.log
+2024-08-12 07:39:55,679 INFO    MainThread:14724 [wandb_init.py:init():566] calling init triggers
+2024-08-12 07:39:55,679 INFO    MainThread:14724 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
+config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'valid_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'test_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 4096, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-qwen2-0.5B_train_2024-08-12-07:39:43', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-qwen2-0.5B', 'save': '/work/llm_recipes/models/yans-qwen2-0.5B', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 10, 'save_interval': 10, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 1, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-qwen2-0.5B', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 320}
+2024-08-12 07:39:55,679 INFO    MainThread:14724 [wandb_init.py:init():616] starting backend
+2024-08-12 07:39:55,679 INFO    MainThread:14724 [wandb_init.py:init():620] setting up manager
+2024-08-12 07:39:55,684 INFO    MainThread:14724 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-08-12 07:39:55,685 INFO    MainThread:14724 [wandb_init.py:init():628] backend started and connected
+2024-08-12 07:39:55,689 INFO    MainThread:14724 [wandb_init.py:init():720] updated telemetry
+2024-08-12 07:39:55,704 INFO    MainThread:14724 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-08-12 07:39:56,202 INFO    MainThread:14724 [wandb_run.py:_on_init():2262] communicating current version
+2024-08-12 07:39:56,287 INFO    MainThread:14724 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.6 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-08-12 07:39:56,287 INFO    MainThread:14724 [wandb_init.py:init():804] starting run threads in backend
+2024-08-12 07:39:56,346 INFO    MainThread:14724 [wandb_run.py:_console_start():2241] atexit reg
+2024-08-12 07:39:56,346 INFO    MainThread:14724 [wandb_run.py:_redirect():2096] redirect: wrap_raw
+2024-08-12 07:39:56,346 INFO    MainThread:14724 [wandb_run.py:_redirect():2161] Wrapping output streams.
+2024-08-12 07:39:56,347 INFO    MainThread:14724 [wandb_run.py:_redirect():2186] Redirects installed.
+2024-08-12 07:39:56,348 INFO    MainThread:14724 [wandb_init.py:init():847] run started, returning control to user process
+2024-08-12 07:40:02,086 INFO    MainThread:14724 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 4096, 'num_attention_heads': 14, 'num_hidden_layers': 24}
+2024-08-12 07:40:02,086 INFO    MainThread:14724 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1}

wandb/run-20240823_160642-78xnl14c/files/config.yaml

@@ -0,0 +1,342 @@
+wandb_version: 1
+
+sharding_strategy:
+  desc: null
+  value: FULL_SHARD
+checkpoint_type:
+  desc: null
+  value: LOCAL_STATE_DICT
+fsdp_activation_checkpointing:
+  desc: null
+  value: true
+fsdp_cpu_offload:
+  desc: null
+  value: false
+low_cpu_fsdp:
+  desc: null
+  value: false
+no_meta_device:
+  desc: null
+  value: false
+data_path:
+  desc: null
+  value: null
+split:
+  desc: null
+  value: 969, 30, 1
+train_data_path:
+  desc: null
+  value:
+  - '1754785366'
+  - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document
+  - '28623823675'
+  - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document
+valid_data_path:
+  desc: null
+  value:
+  - '1754785366'
+  - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document
+test_data_path:
+  desc: null
+  value:
+  - '1754785366'
+  - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document
+data_cache_path:
+  desc: null
+  value: null
+vocab_size:
+  desc: null
+  value: null
+vocab_file:
+  desc: null
+  value: null
+merge_file:
+  desc: null
+  value: null
+seq_length:
+  desc: null
+  value: 2048
+num_workers:
+  desc: null
+  value: 2
+tokenizer_type:
+  desc: null
+  value: HFPreTrainedTokenizer
+tokenizer_model:
+  desc: null
+  value: /share/pretrained_lm/Qwen/Qwen2-0.5B
+reset_position_ids:
+  desc: null
+  value: false
+reset_attention_mask:
+  desc: null
+  value: false
+eod_mask_loss:
+  desc: null
+  value: false
+retro_return_doc_ids:
+  desc: null
+  value: false
+short_seq_prob:
+  desc: null
+  value: 0.1
+vocab_extra_ids:
+  desc: null
+  value: 0
+seed:
+  desc: null
+  value: 1234
+use_mpi:
+  desc: null
+  value: false
+wandb_entity:
+  desc: null
+  value: iwakawa-koichi-q5-tohoku-nlp6723
+wandb_name:
+  desc: null
+  value: Qwen2-0.5b-0.2_train_2024-08-23-16:06:29
+wandb_project:
+  desc: null
+  value: llm_tutorial-0.2
+quantization:
+  desc: null
+  value: false
+use_freeze_layers:
+  desc: null
+  value: false
+freeze_layers:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+mixed_precision:
+  desc: null
+  value: true
+param_dtype:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: /work/llm_recipes/models/Qwen2-0.5b-0.2
+save:
+  desc: null
+  value: /work/llm_recipes/models/Qwen2-0.5b-0.2
+base_model:
+  desc: null
+  value: /share/pretrained_lm/Qwen/Qwen2-0.5B
+use_better_transformer:
+  desc: null
+  value: false
+grad_clip_norm:
+  desc: null
+  value: 1.0
+eval_interval:
+  desc: null
+  value: 10
+save_interval:
+  desc: null
+  value: 10
+eval_iters:
+  desc: null
+  value: 10
+optimizer:
+  desc: null
+  value: anyprecision
+lr:
+  desc: null
+  value: 2.0e-05
+lr_decay_style:
+  desc: null
+  value: cosine
+lr_decay_iters:
+  desc: null
+  value: 7500
+lr_warmup_iters:
+  desc: null
+  value: 500
+min_lr:
+  desc: null
+  value: 1.0e-06
+train_iters:
+  desc: null
+  value: 7500
+train_samples:
+  desc: null
+  value: null
+global_batch_size:
+  desc: null
+  value: 320
+micro_batch_size:
+  desc: null
+  value: 5
+make_vocab_size_divisible_by:
+  desc: null
+  value: 128
+sliding_window_size:
+  desc: null
+  value: 131072
+skip_batch:
+  desc: null
+  value: null
+no_save_optimizer_state:
+  desc: null
+  value: false
+continual_pretraining:
+  desc: null
+  value: false
+instruction_tuning:
+  desc: null
+  value: false
+direct_preference_optimization:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.1
+hidden_dropout:
+  desc: null
+  value: 0.1
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_eps:
+  desc: null
+  value: 1.0e-06
+hf_transformer_model_dir:
+  desc: null
+  value: null
+instruction_train_data_path:
+  desc: null
+  value: null
+instruction_valid_data_path:
+  desc: null
+  value: null
+epoch:
+  desc: null
+  value: null
+instruction_dataset_size:
+  desc: null
+  value: null
+save_sampler_state:
+  desc: null
+  value: false
+label_smoothing:
+  desc: null
+  value: 0.0
+save_n_checkpoints:
+  desc: null
+  value: 10
+hf_repo_id:
+  desc: null
+  value: koichi12/Qwen2-0.5b-0.2
+create_public_hf_repo:
+  desc: null
+  value: false
+upload_all_checkpoints_to_hf:
+  desc: null
+  value: true
+hf_upload_retry_limit:
+  desc: null
+  value: 2
+exit_duration_in_mins:
+  desc: null
+  value: null
+source_key:
+  desc: null
+  value: null
+target_key:
+  desc: null
+  value: null
+attn_implementation:
+  desc: null
+  value: flash_attention_2
+efficient_instruction_tuning:
+  desc: null
+  value: false
+remove_padding_masking:
+  desc: null
+  value: false
+save_start_iter:
+  desc: null
+  value: null
+valid_micro_batch_size:
+  desc: null
+  value: 1
+rank:
+  desc: null
+  value: 0
+world_size:
+  desc: null
+  value: 1
+padded_vocab_size:
+  desc: null
+  value: 151680
+gradient_accumulation_steps:
+  desc: null
+  value: 64
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.43.3
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1724396802.555005
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      - 105
+      2:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      - 105
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.43.3
+      8:
+      - 5
+      13: linux-x86_64
+model_architecture:
+  desc: null
+  value: Qwen2ForCausalLM
+activation_function:
+  desc: null
+  value: silu
+hidden_size:
+  desc: null
+  value: 896
+model_type:
+  desc: null
+  value: qwen2
+max_position_embeddings:
+  desc: null
+  value: 2048
+num_attention_heads:
+  desc: null
+  value: 14
+num_hidden_layers:
+  desc: null
+  value: 24

wandb/run-20240823_160642-78xnl14c/files/output.log

@@ -0,0 +1,253 @@
+Created Hugging Face repository with ID koichi12/Qwen2-0.5b-0.2.
+Clearing GPU cache for all ranks
+--> Running with torch torch_distributed debug set to detail
+Loading model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000010/model.pt
+Loaded model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000010/model.pt
+--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B
+--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params
+BFloat16 enabled for mixed precision - using bfSixteen policy
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1.
+  warnings.warn(
+Let split = None
+Unable to save the indexes because path_to_cache is None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+--> applying fsdp activation checkpointing...
+ > datasets target sizes (minimum size):
+    train:      2400000
+    validation: 2403200
+    test:       3200
+> building train, validation, and test datasets for GPT ...
+> finished creating GPT datasets ...
+Loading optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000010/optimizer.pt
+Loaded optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000010/optimizer.pt
+model info: FullyShardedDataParallel(
+  (_fsdp_wrapped_module): Qwen2ForCausalLM(
+    (model): Qwen2Model(
+      (embed_tokens): Embedding(151936, 896)
+      (layers): ModuleList(
+        (0-23): 24 x FullyShardedDataParallel(
+          (_fsdp_wrapped_module): CheckpointWrapper(
+            (_checkpoint_wrapped_module): Qwen2DecoderLayer(
+              (self_attn): Qwen2FlashAttention2(
+                (q_proj): Linear(in_features=896, out_features=896, bias=True)
+                (k_proj): Linear(in_features=896, out_features=128, bias=True)
+                (v_proj): Linear(in_features=896, out_features=128, bias=True)
+                (o_proj): Linear(in_features=896, out_features=896, bias=False)
+                (rotary_emb): Qwen2RotaryEmbedding()
+              )
+              (mlp): Qwen2MLP(
+                (gate_proj): Linear(in_features=896, out_features=4864, bias=False)
+                (up_proj): Linear(in_features=896, out_features=4864, bias=False)
+                (down_proj): Linear(in_features=4864, out_features=896, bias=False)
+                (act_fn): SiLU()
+              )
+              (input_layernorm): Qwen2RMSNorm()
+              (post_attention_layernorm): Qwen2RMSNorm()
+            )
+          )
+        )
+      )
+      (norm): Qwen2RMSNorm()
+    )
+    (lm_head): Linear(in_features=896, out_features=151936, bias=False)
+  )
+)
+model config: Qwen2Config {
+  "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "label_smoothing": 0.0,
+  "max_position_embeddings": 2048,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.3",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}
+[rank0]:[2024-08-23 16:06:47,708] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling:  defaultdict(<class 'float'>, {})
+------------------------------------------------------------------
+iteration: 11 , TFLOPS: 78.6590174462143, Tokens per sec: 22522.217665366814, Loss: 4.254438400268555
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 12 , TFLOPS: 82.22378668182658, Tokens per sec: 23542.90812474309, Loss: 4.243721008300781
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 13 , TFLOPS: 81.9584539835788, Tokens per sec: 23466.93615131027, Loss: 4.228161334991455
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 14 , TFLOPS: 81.93344116019101, Tokens per sec: 23459.774299165976, Loss: 4.26573371887207
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 15 , TFLOPS: 82.18786108647078, Tokens per sec: 23532.62164895645, Loss: 4.256962776184082
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 16 , TFLOPS: 81.96618258354832, Tokens per sec: 23469.14905984146, Loss: 4.25914192199707
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 17 , TFLOPS: 81.87438758063158, Tokens per sec: 23442.865651995726, Loss: 4.231760025024414
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 18 , TFLOPS: 81.88852755364577, Tokens per sec: 23446.914311136756, Loss: 4.2337870597839355
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 19 , TFLOPS: 82.2474099184413, Tokens per sec: 23549.672100371965, Loss: 4.217740535736084
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 20 , TFLOPS: 81.96982275621532, Tokens per sec: 23470.191340355606, Loss: 4.259789943695068
+------------------------------------------------------------------
+Saving checkpoint to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020
+Saving model state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/model.pt
+/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.
+  warnings.warn(
+/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.
+  warnings.warn(
+Saved model state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/model.pt
+Saving optimizer state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/optimizer.pt
+[rank0]:[2024-08-23 16:11:31,233] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling:  defaultdict(<class 'float'>, {'preprocessing': 0.006403441000202292, 'preprocessing_with_comm': 0.0006401519999599259, 'state_converting': 0.9829786160003096, <Type.ALL: 'all'>: 0.9914544049997858})
+Saved optimizer state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/optimizer.pt
+Saving scheduler state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/scheduler.pt
+Saved scheduler state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/scheduler.pt
+Saving RNG states to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/rng.pt
+Saved RNG states to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020/rng.pt
+Saved checkpoint to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000020, took 4.19s
+ eval ppl=57.051639556884766, eval loss=4.043956756591797
+------------------------------------------------------------------
+iteration: 21 , TFLOPS: 81.67173393242047, Tokens per sec: 23384.840396610194, Loss: 4.235799789428711
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 22 , TFLOPS: 82.17554686977023, Tokens per sec: 23529.095753542137, Loss: 4.2608537673950195
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 23 , TFLOPS: 81.93471100946451, Tokens per sec: 23460.13789157621, Loss: 4.211125373840332
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 24 , TFLOPS: 81.95296715435393, Tokens per sec: 23465.36512276062, Loss: 4.202465534210205
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 25 , TFLOPS: 81.86218501721527, Tokens per sec: 23439.37172595571, Loss: 4.217883586883545
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 26 , TFLOPS: 82.2054193168547, Tokens per sec: 23537.64904822184, Loss: 4.235620021820068
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 27 , TFLOPS: 81.90650355273718, Tokens per sec: 23452.06132895408, Loss: 4.211632251739502
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 28 , TFLOPS: 81.86577887433305, Tokens per sec: 23440.400745067473, Loss: 4.186619758605957
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 29 , TFLOPS: 81.98724461570576, Tokens per sec: 23475.179692922393, Loss: 4.187148571014404
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 30 , TFLOPS: 82.14778410156913, Tokens per sec: 23521.146517348767, Loss: 4.202610492706299
+------------------------------------------------------------------
+Saving checkpoint to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030
+Saving model state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/model.pt
+Saved model state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/model.pt
+Saving optimizer state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/optimizer.pt
+[rank0]:[2024-08-23 16:16:16,915] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling:  defaultdict(<class 'float'>, {'preprocessing': 0.0063674119996903755, 'preprocessing_with_comm': 0.0006565419998878497, 'state_converting': 0.9940128050002386, <Type.ALL: 'all'>: 1.0024299110000356})
+Saved optimizer state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/optimizer.pt
+Saving scheduler state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/scheduler.pt
+Saved scheduler state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/scheduler.pt
+Saving RNG states to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/rng.pt
+Saved RNG states to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030/rng.pt
+Saved checkpoint to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000030, took 4.25s
+ eval ppl=49.190242767333984, eval loss=3.895695209503174
+------------------------------------------------------------------
+iteration: 31 , TFLOPS: 81.49741363388435, Tokens per sec: 23334.92775042431, Loss: 4.200056552886963
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 32 , TFLOPS: 81.78042346796904, Tokens per sec: 23415.961168990125, Loss: 4.181160926818848
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 33 , TFLOPS: 81.92238077096499, Tokens per sec: 23456.60740868476, Loss: 4.155094623565674
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 34 , TFLOPS: 82.11937602485122, Tokens per sec: 23513.012511762787, Loss: 4.177372932434082
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 35 , TFLOPS: 81.71716264856666, Tokens per sec: 23397.847874538613, Loss: 4.142157077789307
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 36 , TFLOPS: 81.89085188246439, Tokens per sec: 23447.57982974198, Loss: 4.166767597198486
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 37 , TFLOPS: 81.95670790833275, Tokens per sec: 23466.436202441997, Loss: 4.169678688049316
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 38 , TFLOPS: 82.18953848455327, Tokens per sec: 23533.101933683516, Loss: 4.145669937133789
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 39 , TFLOPS: 81.94531984778251, Tokens per sec: 23463.175490741974, Loss: 4.136501312255859
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 40 , TFLOPS: 81.88752676960887, Tokens per sec: 23446.627759427276, Loss: 4.12642240524292
+------------------------------------------------------------------
+Saving checkpoint to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040
+Saving model state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt
+Saved model state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt
+[rank0]:[2024-08-23 16:21:02,356] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling:  defaultdict(<class 'float'>, {'preprocessing': 0.006349127999783377, 'preprocessing_with_comm': 0.0006216020001375, 'state_converting': 0.7473425819998738, <Type.ALL: 'all'>: 0.7556547180001871})
+Saving optimizer state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt
+Saved optimizer state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt
+Saving scheduler state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/scheduler.pt
+Saved scheduler state dict to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/scheduler.pt
+Saving RNG states to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/rng.pt
+Saved RNG states to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/rng.pt
+Saved checkpoint to /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040, took 3.86s
+ eval ppl=30.580110549926758, eval loss=3.4203498363494873
+------------------------------------------------------------------
+iteration: 41 , TFLOPS: 81.74253389997872, Tokens per sec: 23405.112354382087, Loss: 4.1325507164001465
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 42 , TFLOPS: 81.9581488586999, Tokens per sec: 23466.848785752438, Loss: 4.099006652832031
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 43 , TFLOPS: 81.91532895406834, Tokens per sec: 23454.588281568267, Loss: 4.136029243469238
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 44 , TFLOPS: 81.95839032196434, Tokens per sec: 23466.917923257286, Loss: 4.140143871307373
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 45 , TFLOPS: 82.23276643961596, Tokens per sec: 23545.47927381259, Loss: 4.161101341247559
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 46 , TFLOPS: 81.96124008786207, Tokens per sec: 23467.733888799798, Loss: 4.099796772003174
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 47 , TFLOPS: 81.96352747485656, Tokens per sec: 23468.388829955275, Loss: 4.10368537902832
+------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/project/examples/finetuning.py", line 13, in <module>
+    main()
+  File "/project/src/llama_recipes/finetuning.py", line 282, in main
+    train(
+  File "/project/src/llama_recipes/utils/train_utils.py", line 118, in train
+    loss.backward()
+  File "/usr/local/lib/python3.10/dist-packages/torch/_tensor.py", line 522, in backward
+    torch.autograd.backward(
+  File "/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py", line 267, in backward
+    _engine_run_backward(
+  File "/usr/local/lib/python3.10/dist-packages/torch/autograd/graph.py", line 681, in _engine_run_backward
+    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+KeyboardInterrupt

wandb/run-20240823_160642-78xnl14c/files/requirements.txt

@@ -0,0 +1,375 @@
+absl-py==2.1.0
+accelerate==0.23.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+apex==0.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+astroid==3.2.4
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+beautifulsoup4==4.12.3
+bert-score==0.3.13
+bleach==6.1.0
+blis==0.7.11
+build==1.2.1
+cachecontrol==0.14.0
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+cleo==2.1.0
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+cmake==3.28.1
+colorama==0.4.6
+comm==0.2.1
+confection==0.1.4
+contourpy==1.2.0
+cramjam==2.8.3
+crashtest==0.4.1
+cryptography==43.0.0
+cubinlinker==0.3.0+2.g405ac64
+cuda-python==12.3.0rc4+9.gdb8c48a.dirty
+cudf==23.12.0
+cugraph-dgl==23.12.0
+cugraph-service-client==23.12.0
+cugraph-service-server==23.12.0
+cugraph==23.12.0
+cuml==23.12.0
+cupy-cuda12x==12.3.0
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.8
+dask-cuda==23.12.0
+dask-cudf==23.12.0
+dask==2023.11.0
+dataclasses-json==0.6.7
+dataproperty==1.0.1
+datasets==2.20.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.8
+distlib==0.3.8
+distributed==2023.11.0
+distro==1.9.0
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+dulwich==0.21.7
+einops==0.7.0
+emoji==2.12.1
+entmax==1.3
+evaluate==0.4.2
+exceptiongroup==1.2.0
+execnet==2.0.2
+executing==2.0.1
+expecttest==0.1.3
+fastjsonschema==2.19.1
+fastparquet==2023.10.1
+fastrlock==0.8.2
+filelock==3.13.1
+flash-attn==2.4.2
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2023.12.2
+fugashi==1.3.2
+fuzzywuzzy==0.18.0
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.27.0
+graphsurgeon==0.4.6
+greenlet==3.0.3
+grpcio==1.60.1
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.24.5
+hydra-core==1.3.2
+hypothesis==5.35.1
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+installer==0.7.0
+intel-openmp==2021.4.0
+ipadic==1.0.0
+ipykernel==6.29.2
+ipython-genutils==0.2.0
+ipython==8.21.0
+isort==5.13.2
+jaraco.classes==3.4.0
+jedi==0.19.1
+jeepney==0.8.0
+jinja2==3.1.3
+jiter==0.5.0
+joblib==1.3.2
+json5==0.9.14
+jsonargparse==3.13.1
+jsonlines==4.0.0
+jsonnet==0.19.1
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema-specifications==2023.12.1
+jsonschema==4.21.1
+jupyter-client==8.6.0
+jupyter-core==5.7.1
+jupyter-tensorboard==0.2.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==1.2.0
+jupyterlab==2.3.2
+jupytext==1.16.1
+keyring==24.3.1
+kiwisolver==1.4.5
+langchain-community==0.2.12
+langchain-core==0.2.31
+langchain-huggingface==0.0.2
+langchain-openai==0.1.21
+langchain-text-splitters==0.2.2
+langchain==0.2.13
+langcodes==3.3.0
+langsmith==0.1.99
+lazy-loader==0.3
+levenshtein==0.25.1
+librosa==0.10.1
+lightning-utilities==0.11.6
+llm-jp-eval==1.4.0
+llvmlite==0.40.1
+lm-eval==0.3.0
+locket==1.0.0
+logzero==1.7.0
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown==3.5.2
+markupsafe==2.1.4
+marshmallow==3.21.3
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mbstrdecoder==1.1.3
+mccabe==0.7.0
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mecab-python3==1.0.6
+mistune==3.0.2
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mkl==2021.1.1
+mock==5.1.0
+mojimoji==0.0.13
+more-itertools==9.1.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+multiprocess==0.70.16
+murmurhash==1.0.10
+mypy-extensions==1.0.0
+nbclient==0.9.0
+nbconvert==7.16.0
+nbformat==5.9.2
+neologdn==0.5.3
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.4.10
+numba==0.57.1+1.g1ff679645
+numexpr==2.10.1
+numpy==1.24.4
+nvfuser==0.1.4a0+d0bb811
+nvidia-dali-cuda120==1.34.0
+nvidia-pyindex==1.0.9
+nvtx==0.2.5
+oauthlib==3.2.2
+omegaconf==2.3.0
+onnx==1.15.0rc2
+openai==1.40.6
+opencv==4.7.0
+optree==0.10.0
+orjson==3.10.7
+packaging==23.2
+pandas==2.2.2
+pandocfilters==1.5.1
+parso==0.8.3
+partd==1.4.1
+pathvalidate==3.2.0
+peft==0.5.0
+pexpect==4.9.0
+pillow==10.2.0
+pip==24.0
+pkginfo==1.11.1
+plac==1.4.3
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+poetry-core==1.9.0
+poetry-plugin-export==1.8.0
+poetry==1.8.3
+polygraphy==0.49.4
+pooch==1.8.0
+portalocker==2.10.1
+preshed==3.0.9
+prettytable==3.9.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.24.4
+psutil==5.9.4
+ptxcompiler==0.8.1+2.g0d406d6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow-hotfix==0.6
+pyarrow==15.0.2
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pybind11-global==2.11.1
+pybind11==2.11.1
+pycocotools==2.0+nv0.8.0
+pycountry==24.6.1
+pycparser==2.21
+pydantic-core==2.16.2
+pydantic==2.6.1
+pygments==2.17.2
+pylibcugraph==23.12.0
+pylibcugraphops==23.12.0
+pylibraft==23.12.0
+pylint==3.2.6
+pynvml==11.4.1
+pyparsing==3.1.1
+pyproject-hooks==1.1.0
+pytablewriter==1.2.0
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==13.0
+pytest-shard==0.1.2
+pytest-xdist==3.5.0
+pytest==8.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-hostlist==1.23.0
+python-levenshtein==0.25.1
+pytorch-lightning==2.4.0
+pytorch-quantization==2.1.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.2
+raft-dask==23.12.0
+rapidfuzz==3.9.6
+rapids-dask-dependency==23.12.1
+referencing==0.33.0
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests-toolbelt==1.0.0
+requests==2.32.3
+rhoknp==1.7.0
+rich==13.7.0
+rmm==23.12.0
+rouge-score==0.1.2
+rpds-py==0.17.1
+rsa==4.9
+sacrebleu==2.4.2
+safetensors==0.4.3
+scikit-learn==1.5.1
+scipy==1.12.0
+secretstorage==3.3.3
+send2trash==1.8.2
+sentence-transformers==3.0.1
+sentencepiece==0.1.99
+sentry-sdk==2.12.0
+setproctitle==1.3.3
+setuptools==68.2.2
+shellingham==1.5.4
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sniffio==1.3.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sphinx-glpi-theme==0.6
+sqlalchemy==2.0.32
+sqlitedict==2.1.0
+srsly==2.4.8
+stack-data==0.6.3
+sumeval==0.2.2
+sympy==1.12
+tabledata==1.3.3
+tabulate==0.9.0
+tbb==2021.11.0
+tblib==3.0.0
+tcolorpy==0.1.6
+tenacity==8.5.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.9.0
+tensorrt==8.6.3
+terminado==0.18.0
+termplotlib==0.3.9
+text-generation==0.7.0
+thinc==8.2.3
+threadpoolctl==3.2.0
+thriftpy2==0.4.17
+tiktoken==0.7.0
+tinycss2==1.2.1
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+tomlkit==0.13.2
+toolz==0.12.1
+torch-tensorrt==2.3.0a0
+torch==2.3.0a0+ebedce2
+torchdata==0.7.1a0
+torchmetrics==0.10.3
+torchtext==0.17.0a0
+torchvision==0.18.0a0
+tornado==6.4
+tqdm-multiprocess==0.0.11
+tqdm==4.66.5
+traitlets==5.9.0
+transformer-engine==1.3.0+5b90b7f
+transformers==4.43.3
+treelite-runtime==3.9.1
+treelite==3.9.1
+triton==2.2.0+e28a256
+trove-classifiers==2024.7.2
+typepy==1.3.2
+typer==0.9.0
+types-dataclasses==0.6.6
+typing-extensions==4.12.2
+typing-inspect==0.9.0
+tzdata==2024.1
+ucx-py==0.35.0
+uff==0.6.9
+ujson==5.8.0
+unbabel-comet==2.2.2
+unidic-lite==1.0.8
+urllib3==1.26.18
+virtualenv==20.26.3
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.3.4
+webencodings==0.5.1
+werkzeug==3.0.1
+wheel==0.42.0
+word2number==1.1
+xdoctest==1.0.2
+xgboost==1.7.6
+xmltodict==0.13.0
+xxhash==3.4.1
+yarl==1.9.4
+zict==3.0.0
+zipp==3.17.0
+zstandard==0.23.0

wandb/run-20240823_160642-78xnl14c/files/wandb-metadata.json

@@ -0,0 +1,220 @@
+{
+    "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-08-23T07:06:43.074166",
+    "startedAt": "2024-08-23T07:06:42.542542",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--seq-length",
+        "2048",
+        "--sliding-window-size",
+        "131072",
+        "--micro-batch-size",
+        "5",
+        "--valid_micro_batch_size",
+        "1",
+        "--global-batch-size",
+        "320",
+        "--train-iters",
+        "7500",
+        "--tokenizer-type",
+        "HFPreTrainedTokenizer",
+        "--tokenizer-model",
+        "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+        "--train-data-path",
+        "1754785366",
+        "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document",
+        "28623823675",
+        "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document",
+        "--valid-data-path",
+        "1754785366",
+        "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document",
+        "--test-data-path",
+        "1754785366",
+        "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document",
+        "--lr",
+        "2e-5",
+        "--min-lr",
+        "1e-6",
+        "--lr-decay-style",
+        "cosine",
+        "--lr-warmup-iters",
+        "500",
+        "--lr-decay-iters",
+        "7500",
+        "--weight-decay",
+        "0.1",
+        "--grad-clip-norm",
+        "1.0",
+        "--optimizer",
+        "anyprecision",
+        "--adam-beta1",
+        "0.9",
+        "--adam-beta2",
+        "0.95",
+        "--adam-eps",
+        "1e-6",
+        "--save-interval",
+        "10",
+        "--eval-interval",
+        "10",
+        "--eval-iters",
+        "10",
+        "--bf16",
+        "--mixed-precision",
+        "--base-model",
+        "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+        "--save",
+        "/work/llm_recipes/models/Qwen2-0.5b-0.2",
+        "--load",
+        "/work/llm_recipes/models/Qwen2-0.5b-0.2",
+        "--fsdp-activation-checkpointing",
+        "--sharding-strategy",
+        "FULL_SHARD",
+        "--checkpoint-type",
+        "LOCAL_STATE_DICT",
+        "--save-n-checkpoints",
+        "10",
+        "--upload-all-checkpoints-to-hf",
+        "--hf-upload-retry-limit",
+        "2",
+        "--hf-repo-id",
+        "koichi12/Qwen2-0.5b-0.2",
+        "--wandb-entity",
+        "iwakawa-koichi-q5-tohoku-nlp6723",
+        "--wandb-project",
+        "llm_tutorial-0.2",
+        "--wandb-name",
+        "Qwen2-0.5b-0.2_train_2024-08-23-16:06:29"
+    ],
+    "state": "running",
+    "program": "/project/examples/finetuning.py",
+    "codePathLocal": "examples/finetuning.py",
+    "codePath": "examples/finetuning.py",
+    "git": {
+        "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git",
+        "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6"
+    },
+    "email": null,
+    "root": "/project",
+    "host": "gpu-koiwa-00",
+    "username": "koiwa",
+    "executable": "/usr/bin/python",
+    "cpu_count": 18,
+    "cpu_count_logical": 18,
+    "cpu_freq": {
+        "current": 2400.0389999999993,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 0.0625,
+            "used": 1.1444091796875e-05
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 56.487831115722656
+    }
+}

wandb/run-20240823_160642-78xnl14c/files/wandb-summary.json

@@ -0,0 +1 @@
+{"training/loss": 4.10368537902832, "training/perplexity": 60.56307470314165, "utils/batch_size": 5, "utils/global_batch_size": 320, "utils/seq_len": 2049, "utils/gradient_accumulation_steps": 64, "utils/iteration": 47, "optimizer/lr": 2.786e-06, "optimizer/variance_l2": 0.05447102242525233, "optimizer/variance_sqrt_l2": 0.9553866833854993, "optimizer/momentum_l2": 0.9463549769133376, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 0.9140548706054688, "optimizer/variance_sqrt_l1": 4069.0, "optimizer/momentum_l1": 3366.75, "optimizer/weight_l1": 6886400.0, "optimizer/variance_abs_max": 0.044921875, "optimizer/variance_sqrt_abs_max": 0.2119140625, "optimizer/momentum_abs_max": 0.2353515625, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 27.938858723999147, "stats/tokens_per_sec": 23468.388829955275, "stats/tokens_per_sec_per_gpu": 23468.388829955275, "stats/tflops": 81.96352747485656, "_timestamp": 1724397861.433749, "_runtime": 1058.8787438869476, "_step": 47, "evaluation/val_loss": 3.4203498363494873, "evaluation/val_ppl": 30.580110549926758, "_wandb": {"runtime": 1064}}

wandb/run-20240823_160642-78xnl14c/logs/debug.log

@@ -0,0 +1,30 @@
+2024-08-23 16:06:42,548 INFO    MainThread:10858 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3
+2024-08-23 16:06:42,549 INFO    MainThread:10858 [wandb_setup.py:_flush():76] Configure stats pid to 10858
+2024-08-23 16:06:42,549 INFO    MainThread:10858 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings
+2024-08-23 16:06:42,549 INFO    MainThread:10858 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings
+2024-08-23 16:06:42,549 INFO    MainThread:10858 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'}
+2024-08-23 16:06:42,549 INFO    MainThread:10858 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-08-23 16:06:42,549 INFO    MainThread:10858 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'}
+2024-08-23 16:06:42,549 INFO    MainThread:10858 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240823_160642-78xnl14c/logs/debug.log
+2024-08-23 16:06:42,549 INFO    MainThread:10858 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240823_160642-78xnl14c/logs/debug-internal.log
+2024-08-23 16:06:42,549 INFO    MainThread:10858 [wandb_init.py:init():566] calling init triggers
+2024-08-23 16:06:42,549 INFO    MainThread:10858 [wandb_init.py:init():573] wandb.init called with sweep_config: {}
+config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'test_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 2048, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'Qwen2-0.5b-0.2_train_2024-08-23-16:06:29', 'wandb_project': 'llm_tutorial-0.2', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'save': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 10, 'save_interval': 10, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 7500, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 7500, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 5, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/Qwen2-0.5b-0.2', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 64}
+2024-08-23 16:06:42,549 INFO    MainThread:10858 [wandb_init.py:init():616] starting backend
+2024-08-23 16:06:42,549 INFO    MainThread:10858 [wandb_init.py:init():620] setting up manager
+2024-08-23 16:06:42,554 INFO    MainThread:10858 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-08-23 16:06:42,554 INFO    MainThread:10858 [wandb_init.py:init():628] backend started and connected
+2024-08-23 16:06:42,559 INFO    MainThread:10858 [wandb_init.py:init():720] updated telemetry
+2024-08-23 16:06:42,570 INFO    MainThread:10858 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-08-23 16:06:42,985 INFO    MainThread:10858 [wandb_run.py:_on_init():2262] communicating current version
+2024-08-23 16:06:43,007 INFO    MainThread:10858 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-08-23 16:06:43,007 INFO    MainThread:10858 [wandb_init.py:init():804] starting run threads in backend
+2024-08-23 16:06:43,104 INFO    MainThread:10858 [wandb_run.py:_console_start():2241] atexit reg
+2024-08-23 16:06:43,105 INFO    MainThread:10858 [wandb_run.py:_redirect():2096] redirect: wrap_raw
+2024-08-23 16:06:43,105 INFO    MainThread:10858 [wandb_run.py:_redirect():2161] Wrapping output streams.
+2024-08-23 16:06:43,105 INFO    MainThread:10858 [wandb_run.py:_redirect():2186] Redirects installed.
+2024-08-23 16:06:43,106 INFO    MainThread:10858 [wandb_init.py:init():847] run started, returning control to user process
+2024-08-23 16:06:47,996 INFO    MainThread:10858 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 2048, 'num_attention_heads': 14, 'num_hidden_layers': 24}
+2024-08-23 16:06:47,997 INFO    MainThread:10858 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1}
+2024-08-23 16:24:34,131 WARNING MsgRouterThr:10858 [router.py:message_loop():77] message_loop has been closed

wandb/run-20240823_162922-z3gs82jm/files/config.yaml

@@ -0,0 +1,342 @@
+wandb_version: 1
+
+sharding_strategy:
+  desc: null
+  value: FULL_SHARD
+checkpoint_type:
+  desc: null
+  value: LOCAL_STATE_DICT
+fsdp_activation_checkpointing:
+  desc: null
+  value: true
+fsdp_cpu_offload:
+  desc: null
+  value: false
+low_cpu_fsdp:
+  desc: null
+  value: false
+no_meta_device:
+  desc: null
+  value: false
+data_path:
+  desc: null
+  value: null
+split:
+  desc: null
+  value: 969, 30, 1
+train_data_path:
+  desc: null
+  value:
+  - '1754785366'
+  - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document
+  - '28623823675'
+  - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document
+valid_data_path:
+  desc: null
+  value:
+  - '1754785366'
+  - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document
+test_data_path:
+  desc: null
+  value:
+  - '1754785366'
+  - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document
+data_cache_path:
+  desc: null
+  value: null
+vocab_size:
+  desc: null
+  value: null
+vocab_file:
+  desc: null
+  value: null
+merge_file:
+  desc: null
+  value: null
+seq_length:
+  desc: null
+  value: 2048
+num_workers:
+  desc: null
+  value: 2
+tokenizer_type:
+  desc: null
+  value: HFPreTrainedTokenizer
+tokenizer_model:
+  desc: null
+  value: /share/pretrained_lm/Qwen/Qwen2-0.5B
+reset_position_ids:
+  desc: null
+  value: false
+reset_attention_mask:
+  desc: null
+  value: false
+eod_mask_loss:
+  desc: null
+  value: false
+retro_return_doc_ids:
+  desc: null
+  value: false
+short_seq_prob:
+  desc: null
+  value: 0.1
+vocab_extra_ids:
+  desc: null
+  value: 0
+seed:
+  desc: null
+  value: 1234
+use_mpi:
+  desc: null
+  value: false
+wandb_entity:
+  desc: null
+  value: iwakawa-koichi-q5-tohoku-nlp6723
+wandb_name:
+  desc: null
+  value: Qwen2-0.5b-0.2_train_2024-08-23-16:29:10
+wandb_project:
+  desc: null
+  value: llm_tutorial-0.2
+quantization:
+  desc: null
+  value: false
+use_freeze_layers:
+  desc: null
+  value: false
+freeze_layers:
+  desc: null
+  value: null
+bf16:
+  desc: null
+  value: true
+fp16:
+  desc: null
+  value: false
+mixed_precision:
+  desc: null
+  value: true
+param_dtype:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: /work/llm_recipes/models/Qwen2-0.5b-0.2
+save:
+  desc: null
+  value: /work/llm_recipes/models/Qwen2-0.5b-0.2
+base_model:
+  desc: null
+  value: /share/pretrained_lm/Qwen/Qwen2-0.5B
+use_better_transformer:
+  desc: null
+  value: false
+grad_clip_norm:
+  desc: null
+  value: 1.0
+eval_interval:
+  desc: null
+  value: 10
+save_interval:
+  desc: null
+  value: 10
+eval_iters:
+  desc: null
+  value: 10
+optimizer:
+  desc: null
+  value: anyprecision
+lr:
+  desc: null
+  value: 2.0e-05
+lr_decay_style:
+  desc: null
+  value: cosine
+lr_decay_iters:
+  desc: null
+  value: 7500
+lr_warmup_iters:
+  desc: null
+  value: 500
+min_lr:
+  desc: null
+  value: 1.0e-06
+train_iters:
+  desc: null
+  value: 7500
+train_samples:
+  desc: null
+  value: null
+global_batch_size:
+  desc: null
+  value: 640
+micro_batch_size:
+  desc: null
+  value: 5
+make_vocab_size_divisible_by:
+  desc: null
+  value: 128
+sliding_window_size:
+  desc: null
+  value: 131072
+skip_batch:
+  desc: null
+  value: null
+no_save_optimizer_state:
+  desc: null
+  value: false
+continual_pretraining:
+  desc: null
+  value: false
+instruction_tuning:
+  desc: null
+  value: false
+direct_preference_optimization:
+  desc: null
+  value: false
+attention_dropout:
+  desc: null
+  value: 0.1
+hidden_dropout:
+  desc: null
+  value: 0.1
+weight_decay:
+  desc: null
+  value: 0.1
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.95
+adam_eps:
+  desc: null
+  value: 1.0e-06
+hf_transformer_model_dir:
+  desc: null
+  value: null
+instruction_train_data_path:
+  desc: null
+  value: null
+instruction_valid_data_path:
+  desc: null
+  value: null
+epoch:
+  desc: null
+  value: null
+instruction_dataset_size:
+  desc: null
+  value: null
+save_sampler_state:
+  desc: null
+  value: false
+label_smoothing:
+  desc: null
+  value: 0.0
+save_n_checkpoints:
+  desc: null
+  value: 10
+hf_repo_id:
+  desc: null
+  value: koichi12/Qwen2-0.5b-0.2
+create_public_hf_repo:
+  desc: null
+  value: false
+upload_all_checkpoints_to_hf:
+  desc: null
+  value: true
+hf_upload_retry_limit:
+  desc: null
+  value: 2
+exit_duration_in_mins:
+  desc: null
+  value: null
+source_key:
+  desc: null
+  value: null
+target_key:
+  desc: null
+  value: null
+attn_implementation:
+  desc: null
+  value: flash_attention_2
+efficient_instruction_tuning:
+  desc: null
+  value: false
+remove_padding_masking:
+  desc: null
+  value: false
+save_start_iter:
+  desc: null
+  value: null
+valid_micro_batch_size:
+  desc: null
+  value: 1
+rank:
+  desc: null
+  value: 0
+world_size:
+  desc: null
+  value: 1
+padded_vocab_size:
+  desc: null
+  value: 151680
+gradient_accumulation_steps:
+  desc: null
+  value: 128
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.12
+    cli_version: 0.16.3
+    framework: huggingface
+    huggingface_version: 4.43.3
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1724398162.884223
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      - 105
+      2:
+      - 1
+      - 11
+      - 49
+      - 55
+      - 71
+      - 105
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.12
+      5: 0.16.3
+      6: 4.43.3
+      8:
+      - 5
+      13: linux-x86_64
+model_architecture:
+  desc: null
+  value: Qwen2ForCausalLM
+activation_function:
+  desc: null
+  value: silu
+hidden_size:
+  desc: null
+  value: 896
+model_type:
+  desc: null
+  value: qwen2
+max_position_embeddings:
+  desc: null
+  value: 2048
+num_attention_heads:
+  desc: null
+  value: 14
+num_hidden_layers:
+  desc: null
+  value: 24

wandb/run-20240823_162922-z3gs82jm/files/output.log

@@ -0,0 +1,174 @@
+Created Hugging Face repository with ID koichi12/Qwen2-0.5b-0.2.
+Clearing GPU cache for all ranks
+--> Running with torch torch_distributed debug set to detail
+Loading model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt
+Loaded model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt
+--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B
+--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params
+BFloat16 enabled for mixed precision - using bfSixteen policy
+You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`.
+/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1.
+  warnings.warn(
+Let split = None
+--> applying fsdp activation checkpointing...
+ > datasets target sizes (minimum size):
+    train:      4800000
+    validation: 4806400
+    test:       6400
+> building train, validation, and test datasets for GPT ...
+Unable to save the indexes because path_to_cache is None
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+> finished creating GPT datasets ...
+Loading optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt
+Building a BlendedDataset for a single MegatronDataset
+Unable to save the indexes because path_to_cache is None
+Loaded optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt
+model info: FullyShardedDataParallel(
+  (_fsdp_wrapped_module): Qwen2ForCausalLM(
+    (model): Qwen2Model(
+      (embed_tokens): Embedding(151936, 896)
+      (layers): ModuleList(
+        (0-23): 24 x FullyShardedDataParallel(
+          (_fsdp_wrapped_module): CheckpointWrapper(
+            (_checkpoint_wrapped_module): Qwen2DecoderLayer(
+              (self_attn): Qwen2FlashAttention2(
+                (q_proj): Linear(in_features=896, out_features=896, bias=True)
+                (k_proj): Linear(in_features=896, out_features=128, bias=True)
+                (v_proj): Linear(in_features=896, out_features=128, bias=True)
+                (o_proj): Linear(in_features=896, out_features=896, bias=False)
+                (rotary_emb): Qwen2RotaryEmbedding()
+              )
+              (mlp): Qwen2MLP(
+                (gate_proj): Linear(in_features=896, out_features=4864, bias=False)
+                (up_proj): Linear(in_features=896, out_features=4864, bias=False)
+                (down_proj): Linear(in_features=4864, out_features=896, bias=False)
+                (act_fn): SiLU()
+              )
+              (input_layernorm): Qwen2RMSNorm()
+              (post_attention_layernorm): Qwen2RMSNorm()
+            )
+          )
+        )
+      )
+      (norm): Qwen2RMSNorm()
+    )
+    (lm_head): Linear(in_features=896, out_features=151936, bias=False)
+  )
+)
+model config: Qwen2Config {
+  "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "label_smoothing": 0.0,
+  "max_position_embeddings": 2048,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.3",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}
+[rank0]:[2024-08-23 16:29:30,218] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling:  defaultdict(<class 'float'>, {})
+------------------------------------------------------------------
+iteration: 41 , TFLOPS: 80.56174819694237, Tokens per sec: 23067.0212685365, Loss: 4.141458988189697
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 42 , TFLOPS: 82.11115614789513, Tokens per sec: 23510.658937258577, Loss: 4.13422155380249
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 43 , TFLOPS: 82.06292558214139, Tokens per sec: 23496.84921352573, Loss: 4.125084400177002
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 44 , TFLOPS: 82.21135543718982, Tokens per sec: 23539.348721045328, Loss: 4.142415523529053
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 45 , TFLOPS: 82.19624054931023, Tokens per sec: 23535.020917242735, Loss: 4.127298831939697
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 46 , TFLOPS: 82.08863919564567, Tokens per sec: 23504.21172095474, Loss: 4.131596565246582
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 47 , TFLOPS: 82.19786737309032, Tokens per sec: 23535.486721170473, Loss: 4.140783786773682
+------------------------------------------------------------------
+------------------------------------------------------------------
+iteration: 48 , TFLOPS: 82.07990700682468, Tokens per sec: 23501.711457619313, Loss: 4.111098289489746
+------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/project/examples/finetuning.py", line 13, in <module>
+    main()
+  File "/project/src/llama_recipes/finetuning.py", line 282, in main
+    train(
+  File "/project/src/llama_recipes/utils/train_utils.py", line 110, in train
+    loss: torch.Tensor = model(**batch).loss
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 849, in forward
+    output = self._fsdp_wrapped_module(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/project/lib/transformers/src/transformers/models/qwen2/modeling_qwen2.py", line 1054, in forward
+    outputs = self.model(
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/project/lib/transformers/src/transformers/models/qwen2/modeling_qwen2.py", line 856, in forward
+    layer_outputs = decoder_layer(
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 849, in forward
+    output = self._fsdp_wrapped_module(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/distributed/algorithms/_checkpoint/checkpoint_wrapper.py", line 168, in forward
+    return self.checkpoint_fn(  # type: ignore[misc]
+  File "/usr/local/lib/python3.10/dist-packages/torch/_compile.py", line 24, in inner
+    return torch._dynamo.disable(fn, recursive)(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/_dynamo/eval_frame.py", line 417, in _fn
+    return fn(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/_dynamo/external_utils.py", line 25, in inner
+    return fn(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py", line 488, in checkpoint
+    ret = function(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/project/lib/transformers/src/transformers/models/qwen2/modeling_qwen2.py", line 609, in forward
+    hidden_states = self.post_attention_layernorm(hidden_states)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/project/lib/transformers/src/transformers/models/qwen2/modeling_qwen2.py", line 78, in forward
+    hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+  File "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py", line 1091, in pack_hook
+    with torch.no_grad():
+  File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 149, in __new__
+    def __new__(cls, orig_func=None):
+KeyboardInterrupt

wandb/run-20240823_162922-z3gs82jm/files/requirements.txt

@@ -0,0 +1,375 @@
+absl-py==2.1.0
+accelerate==0.23.0
+aiohttp==3.9.1
+aiosignal==1.3.1
+annotated-types==0.6.0
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+apex==0.1
+appdirs==1.4.4
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+astroid==3.2.4
+asttokens==2.4.1
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+beautifulsoup4==4.12.3
+bert-score==0.3.13
+bleach==6.1.0
+blis==0.7.11
+build==1.2.1
+cachecontrol==0.14.0
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2024.2.2
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+cleo==2.1.0
+click==8.1.7
+cloudpathlib==0.16.0
+cloudpickle==3.0.0
+cmake==3.28.1
+colorama==0.4.6
+comm==0.2.1
+confection==0.1.4
+contourpy==1.2.0
+cramjam==2.8.3
+crashtest==0.4.1
+cryptography==43.0.0
+cubinlinker==0.3.0+2.g405ac64
+cuda-python==12.3.0rc4+9.gdb8c48a.dirty
+cudf==23.12.0
+cugraph-dgl==23.12.0
+cugraph-service-client==23.12.0
+cugraph-service-server==23.12.0
+cugraph==23.12.0
+cuml==23.12.0
+cupy-cuda12x==12.3.0
+cycler==0.12.1
+cymem==2.0.8
+cython==3.0.8
+dask-cuda==23.12.0
+dask-cudf==23.12.0
+dask==2023.11.0
+dataclasses-json==0.6.7
+dataproperty==1.0.1
+datasets==2.20.0
+debugpy==1.8.1
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.8
+distlib==0.3.8
+distributed==2023.11.0
+distro==1.9.0
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+dulwich==0.21.7
+einops==0.7.0
+emoji==2.12.1
+entmax==1.3
+evaluate==0.4.2
+exceptiongroup==1.2.0
+execnet==2.0.2
+executing==2.0.1
+expecttest==0.1.3
+fastjsonschema==2.19.1
+fastparquet==2023.10.1
+fastrlock==0.8.2
+filelock==3.13.1
+flash-attn==2.4.2
+fonttools==4.48.1
+frozenlist==1.4.1
+fsspec==2023.12.2
+fugashi==1.3.2
+fuzzywuzzy==0.18.0
+gast==0.5.4
+gitdb==4.0.11
+gitpython==3.1.43
+google-auth-oauthlib==0.4.6
+google-auth==2.27.0
+graphsurgeon==0.4.6
+greenlet==3.0.3
+grpcio==1.60.1
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.24.5
+hydra-core==1.3.2
+hypothesis==5.35.1
+idna==3.6
+importlib-metadata==7.0.1
+iniconfig==2.0.0
+installer==0.7.0
+intel-openmp==2021.4.0
+ipadic==1.0.0
+ipykernel==6.29.2
+ipython-genutils==0.2.0
+ipython==8.21.0
+isort==5.13.2
+jaraco.classes==3.4.0
+jedi==0.19.1
+jeepney==0.8.0
+jinja2==3.1.3
+jiter==0.5.0
+joblib==1.3.2
+json5==0.9.14
+jsonargparse==3.13.1
+jsonlines==4.0.0
+jsonnet==0.19.1
+jsonpatch==1.33
+jsonpointer==3.0.0
+jsonschema-specifications==2023.12.1
+jsonschema==4.21.1
+jupyter-client==8.6.0
+jupyter-core==5.7.1
+jupyter-tensorboard==0.2.0
+jupyterlab-pygments==0.3.0
+jupyterlab-server==1.2.0
+jupyterlab==2.3.2
+jupytext==1.16.1
+keyring==24.3.1
+kiwisolver==1.4.5
+langchain-community==0.2.12
+langchain-core==0.2.31
+langchain-huggingface==0.0.2
+langchain-openai==0.1.21
+langchain-text-splitters==0.2.2
+langchain==0.2.13
+langcodes==3.3.0
+langsmith==0.1.99
+lazy-loader==0.3
+levenshtein==0.25.1
+librosa==0.10.1
+lightning-utilities==0.11.6
+llm-jp-eval==1.4.0
+llvmlite==0.40.1
+lm-eval==0.3.0
+locket==1.0.0
+logzero==1.7.0
+lxml==5.2.2
+markdown-it-py==3.0.0
+markdown==3.5.2
+markupsafe==2.1.4
+marshmallow==3.21.3
+matplotlib-inline==0.1.6
+matplotlib==3.8.2
+mbstrdecoder==1.1.3
+mccabe==0.7.0
+mdit-py-plugins==0.4.0
+mdurl==0.1.2
+mecab-python3==1.0.6
+mistune==3.0.2
+mkl-devel==2021.1.1
+mkl-include==2021.1.1
+mkl==2021.1.1
+mock==5.1.0
+mojimoji==0.0.13
+more-itertools==9.1.0
+mpmath==1.3.0
+msgpack==1.0.7
+multidict==6.0.4
+multiprocess==0.70.16
+murmurhash==1.0.10
+mypy-extensions==1.0.0
+nbclient==0.9.0
+nbconvert==7.16.0
+nbformat==5.9.2
+neologdn==0.5.3
+nest-asyncio==1.6.0
+networkx==2.6.3
+ninja==1.11.1.1
+nltk==3.8.1
+notebook==6.4.10
+numba==0.57.1+1.g1ff679645
+numexpr==2.10.1
+numpy==1.24.4
+nvfuser==0.1.4a0+d0bb811
+nvidia-dali-cuda120==1.34.0
+nvidia-pyindex==1.0.9
+nvtx==0.2.5
+oauthlib==3.2.2
+omegaconf==2.3.0
+onnx==1.15.0rc2
+openai==1.40.6
+opencv==4.7.0
+optree==0.10.0
+orjson==3.10.7
+packaging==23.2
+pandas==2.2.2
+pandocfilters==1.5.1
+parso==0.8.3
+partd==1.4.1
+pathvalidate==3.2.0
+peft==0.5.0
+pexpect==4.9.0
+pillow==10.2.0
+pip==24.0
+pkginfo==1.11.1
+plac==1.4.3
+platformdirs==4.2.0
+pluggy==1.4.0
+ply==3.11
+poetry-core==1.9.0
+poetry-plugin-export==1.8.0
+poetry==1.8.3
+polygraphy==0.49.4
+pooch==1.8.0
+portalocker==2.10.1
+preshed==3.0.9
+prettytable==3.9.0
+prometheus-client==0.19.0
+prompt-toolkit==3.0.43
+protobuf==4.24.4
+psutil==5.9.4
+ptxcompiler==0.8.1+2.g0d406d6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow-hotfix==0.6
+pyarrow==15.0.2
+pyasn1-modules==0.3.0
+pyasn1==0.5.1
+pybind11-global==2.11.1
+pybind11==2.11.1
+pycocotools==2.0+nv0.8.0
+pycountry==24.6.1
+pycparser==2.21
+pydantic-core==2.16.2
+pydantic==2.6.1
+pygments==2.17.2
+pylibcugraph==23.12.0
+pylibcugraphops==23.12.0
+pylibraft==23.12.0
+pylint==3.2.6
+pynvml==11.4.1
+pyparsing==3.1.1
+pyproject-hooks==1.1.0
+pytablewriter==1.2.0
+pytest-flakefinder==1.1.0
+pytest-rerunfailures==13.0
+pytest-shard==0.1.2
+pytest-xdist==3.5.0
+pytest==8.0.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-hostlist==1.23.0
+python-levenshtein==0.25.1
+pytorch-lightning==2.4.0
+pytorch-quantization==2.1.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.2
+raft-dask==23.12.0
+rapidfuzz==3.9.6
+rapids-dask-dependency==23.12.1
+referencing==0.33.0
+regex==2023.12.25
+requests-oauthlib==1.3.1
+requests-toolbelt==1.0.0
+requests==2.32.3
+rhoknp==1.7.0
+rich==13.7.0
+rmm==23.12.0
+rouge-score==0.1.2
+rpds-py==0.17.1
+rsa==4.9
+sacrebleu==2.4.2
+safetensors==0.4.3
+scikit-learn==1.5.1
+scipy==1.12.0
+secretstorage==3.3.3
+send2trash==1.8.2
+sentence-transformers==3.0.1
+sentencepiece==0.1.99
+sentry-sdk==2.12.0
+setproctitle==1.3.3
+setuptools==68.2.2
+shellingham==1.5.4
+six==1.16.0
+smart-open==6.4.0
+smmap==5.0.1
+sniffio==1.3.1
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soupsieve==2.5
+soxr==0.3.7
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+spacy==3.7.2
+sphinx-glpi-theme==0.6
+sqlalchemy==2.0.32
+sqlitedict==2.1.0
+srsly==2.4.8
+stack-data==0.6.3
+sumeval==0.2.2
+sympy==1.12
+tabledata==1.3.3
+tabulate==0.9.0
+tbb==2021.11.0
+tblib==3.0.0
+tcolorpy==0.1.6
+tenacity==8.5.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorboard==2.9.0
+tensorrt==8.6.3
+terminado==0.18.0
+termplotlib==0.3.9
+text-generation==0.7.0
+thinc==8.2.3
+threadpoolctl==3.2.0
+thriftpy2==0.4.17
+tiktoken==0.7.0
+tinycss2==1.2.1
+tokenizers==0.19.1
+toml==0.10.2
+tomli==2.0.1
+tomlkit==0.13.2
+toolz==0.12.1
+torch-tensorrt==2.3.0a0
+torch==2.3.0a0+ebedce2
+torchdata==0.7.1a0
+torchmetrics==0.10.3
+torchtext==0.17.0a0
+torchvision==0.18.0a0
+tornado==6.4
+tqdm-multiprocess==0.0.11
+tqdm==4.66.5
+traitlets==5.9.0
+transformer-engine==1.3.0+5b90b7f
+transformers==4.43.3
+treelite-runtime==3.9.1
+treelite==3.9.1
+triton==2.2.0+e28a256
+trove-classifiers==2024.7.2
+typepy==1.3.2
+typer==0.9.0
+types-dataclasses==0.6.6
+typing-extensions==4.12.2
+typing-inspect==0.9.0
+tzdata==2024.1
+ucx-py==0.35.0
+uff==0.6.9
+ujson==5.8.0
+unbabel-comet==2.2.2
+unidic-lite==1.0.8
+urllib3==1.26.18
+virtualenv==20.26.3
+wandb==0.16.3
+wasabi==1.1.2
+wcwidth==0.2.13
+weasel==0.3.4
+webencodings==0.5.1
+werkzeug==3.0.1
+wheel==0.42.0
+word2number==1.1
+xdoctest==1.0.2
+xgboost==1.7.6
+xmltodict==0.13.0
+xxhash==3.4.1
+yarl==1.9.4
+zict==3.0.0
+zipp==3.17.0
+zstandard==0.23.0

wandb/run-20240823_162922-z3gs82jm/files/wandb-metadata.json

@@ -0,0 +1,220 @@
+{
+    "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35",
+    "python": "3.10.12",
+    "heartbeatAt": "2024-08-23T07:29:23.385958",
+    "startedAt": "2024-08-23T07:29:22.871856",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "--seq-length",
+        "2048",
+        "--sliding-window-size",
+        "131072",
+        "--micro-batch-size",
+        "5",
+        "--valid_micro_batch_size",
+        "1",
+        "--global-batch-size",
+        "640",
+        "--train-iters",
+        "7500",
+        "--tokenizer-type",
+        "HFPreTrainedTokenizer",
+        "--tokenizer-model",
+        "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+        "--train-data-path",
+        "1754785366",
+        "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document",
+        "28623823675",
+        "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document",
+        "--valid-data-path",
+        "1754785366",
+        "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document",
+        "--test-data-path",
+        "1754785366",
+        "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document",
+        "--lr",
+        "2e-5",
+        "--min-lr",
+        "1e-6",
+        "--lr-decay-style",
+        "cosine",
+        "--lr-warmup-iters",
+        "500",
+        "--lr-decay-iters",
+        "7500",
+        "--weight-decay",
+        "0.1",
+        "--grad-clip-norm",
+        "1.0",
+        "--optimizer",
+        "anyprecision",
+        "--adam-beta1",
+        "0.9",
+        "--adam-beta2",
+        "0.95",
+        "--adam-eps",
+        "1e-6",
+        "--save-interval",
+        "10",
+        "--eval-interval",
+        "10",
+        "--eval-iters",
+        "10",
+        "--bf16",
+        "--mixed-precision",
+        "--base-model",
+        "/share/pretrained_lm/Qwen/Qwen2-0.5B",
+        "--save",
+        "/work/llm_recipes/models/Qwen2-0.5b-0.2",
+        "--load",
+        "/work/llm_recipes/models/Qwen2-0.5b-0.2",
+        "--fsdp-activation-checkpointing",
+        "--sharding-strategy",
+        "FULL_SHARD",
+        "--checkpoint-type",
+        "LOCAL_STATE_DICT",
+        "--save-n-checkpoints",
+        "10",
+        "--upload-all-checkpoints-to-hf",
+        "--hf-upload-retry-limit",
+        "2",
+        "--hf-repo-id",
+        "koichi12/Qwen2-0.5b-0.2",
+        "--wandb-entity",
+        "iwakawa-koichi-q5-tohoku-nlp6723",
+        "--wandb-project",
+        "llm_tutorial-0.2",
+        "--wandb-name",
+        "Qwen2-0.5b-0.2_train_2024-08-23-16:29:10"
+    ],
+    "state": "running",
+    "program": "/project/examples/finetuning.py",
+    "codePathLocal": "examples/finetuning.py",
+    "codePath": "examples/finetuning.py",
+    "git": {
+        "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git",
+        "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6"
+    },
+    "email": null,
+    "root": "/project",
+    "host": "gpu-koiwa-00",
+    "username": "koiwa",
+    "executable": "/usr/bin/python",
+    "cpu_count": 18,
+    "cpu_count_logical": 18,
+    "cpu_freq": {
+        "current": 2400.0389999999993,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2400.039,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 0.0625,
+            "used": 1.1444091796875e-05
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 56.487831115722656
+    }
+}

wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json

@@ -0,0 +1 @@
+{"training/loss": 4.111098289489746, "training/perplexity": 61.013691480602496, "utils/batch_size": 5, "utils/global_batch_size": 640, "utils/seq_len": 2049, "utils/gradient_accumulation_steps": 128, "utils/iteration": 48, "optimizer/lr": 2.8240000000000004e-06, "optimizer/variance_l2": 0.05465312149531553, "optimizer/variance_sqrt_l2": 0.9576321330918345, "optimizer/momentum_l2": 0.9493419425990095, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 0.91644287109375, "optimizer/variance_sqrt_l1": 3987.5, "optimizer/momentum_l1": 3375.75, "optimizer/weight_l1": 6886400.0, "optimizer/variance_abs_max": 0.044921875, "optimizer/variance_sqrt_abs_max": 0.2119140625, "optimizer/momentum_abs_max": 0.234375, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 55.79848949999996, "stats/tokens_per_sec": 23501.711457619313, "stats/tokens_per_sec_per_gpu": 23501.711457619313, "stats/tflops": 82.07990700682468, "_timestamp": 1724398618.465386, "_runtime": 455.5811629295349, "_step": 48, "_wandb": {"runtime": 505}}

wandb/run-20240823_162922-z3gs82jm/logs/debug-internal.log

@@ -0,0 +1,453 @@
+2024-08-23 16:29:22,886 INFO    StreamThr :11966 [internal.py:wandb_internal():86] W&B internal server running at pid: 11966, started at: 2024-08-23 16:29:22.885536
+2024-08-23 16:29:22,888 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status
+2024-08-23 16:29:22,889 INFO    WriterThread:11966 [datastore.py:open_for_write():87] open: /project/wandb/run-20240823_162922-z3gs82jm/run-z3gs82jm.wandb
+2024-08-23 16:29:22,890 DEBUG   SenderThread:11966 [sender.py:send():382] send: header
+2024-08-23 16:29:22,904 DEBUG   SenderThread:11966 [sender.py:send():382] send: run
+2024-08-23 16:29:23,294 INFO    SenderThread:11966 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240823_162922-z3gs82jm/files
+2024-08-23 16:29:23,294 INFO    SenderThread:11966 [sender.py:_start_run_threads():1136] run started: z3gs82jm with start time 1724398162.884223
+2024-08-23 16:29:23,300 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: check_version
+2024-08-23 16:29:23,300 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: check_version
+2024-08-23 16:29:23,367 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: run_start
+2024-08-23 16:29:23,374 DEBUG   HandlerThread:11966 [system_info.py:__init__():27] System info init
+2024-08-23 16:29:23,374 DEBUG   HandlerThread:11966 [system_info.py:__init__():42] System info init done
+2024-08-23 16:29:23,374 INFO    HandlerThread:11966 [system_monitor.py:start():194] Starting system monitor
+2024-08-23 16:29:23,374 INFO    SystemMonitor:11966 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-08-23 16:29:23,374 INFO    HandlerThread:11966 [system_monitor.py:probe():214] Collecting system info
+2024-08-23 16:29:23,374 INFO    SystemMonitor:11966 [interfaces.py:start():190] Started cpu monitoring
+2024-08-23 16:29:23,375 INFO    SystemMonitor:11966 [interfaces.py:start():190] Started disk monitoring
+2024-08-23 16:29:23,375 INFO    SystemMonitor:11966 [interfaces.py:start():190] Started gpu monitoring
+2024-08-23 16:29:23,377 INFO    SystemMonitor:11966 [interfaces.py:start():190] Started memory monitoring
+2024-08-23 16:29:23,378 INFO    SystemMonitor:11966 [interfaces.py:start():190] Started network monitoring
+2024-08-23 16:29:23,385 DEBUG   HandlerThread:11966 [system_info.py:probe():151] Probing system
+2024-08-23 16:29:23,387 DEBUG   HandlerThread:11966 [system_info.py:_probe_git():136] Probing git
+2024-08-23 16:29:23,400 DEBUG   HandlerThread:11966 [system_info.py:_probe_git():144] Probing git done
+2024-08-23 16:29:23,400 DEBUG   HandlerThread:11966 [system_info.py:probe():199] Probing system done
+2024-08-23 16:29:23,400 DEBUG   HandlerThread:11966 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-23T07:29:23.385958', 'startedAt': '2024-08-23T07:29:22.871856', 'docker': None, 'cuda': None, 'args': ('--seq-length', '2048', '--sliding-window-size', '131072', '--micro-batch-size', '5', '--valid_micro_batch_size', '1', '--global-batch-size', '640', '--train-iters', '7500', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document', '--valid-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--test-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '7500', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '10', '--eval-interval', '10', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--load', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/Qwen2-0.5b-0.2', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial-0.2', '--wandb-name', 'Qwen2-0.5b-0.2_train_2024-08-23-16:29:10'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '887a2cc5d104c10264701f95cbbb0a6a116768d6'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0389999999993, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.487831115722656}}
+2024-08-23 16:29:23,400 INFO    HandlerThread:11966 [system_monitor.py:probe():224] Finished collecting system info
+2024-08-23 16:29:23,400 INFO    HandlerThread:11966 [system_monitor.py:probe():227] Publishing system info
+2024-08-23 16:29:23,402 INFO    HandlerThread:11966 [system_monitor.py:probe():229] Finished publishing system info
+2024-08-23 16:29:23,407 DEBUG   SenderThread:11966 [sender.py:send():382] send: files
+2024-08-23 16:29:23,407 INFO    SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now
+2024-08-23 16:29:23,418 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: python_packages
+2024-08-23 16:29:23,419 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:29:23,419 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:29:23,419 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: python_packages
+2024-08-23 16:29:23,421 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:29:23,617 DEBUG   SenderThread:11966 [sender.py:send():382] send: telemetry
+2024-08-23 16:29:24,296 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:29:24,296 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162922-z3gs82jm/files/requirements.txt
+2024-08-23 16:29:24,297 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-metadata.json
+2024-08-23 16:29:24,474 INFO    wandb-upload_0:11966 [upload_job.py:push():131] Uploaded file /tmp/tmpcv014twmwandb/xf5mvw68-wandb-metadata.json
+2024-08-23 16:29:26,296 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:29:28,298 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:29:28,658 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:29:30,299 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:29:30,499 DEBUG   SenderThread:11966 [sender.py:send():382] send: config
+2024-08-23 16:29:30,500 DEBUG   SenderThread:11966 [sender.py:send():382] send: config
+2024-08-23 16:29:32,300 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:29:34,500 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:29:38,418 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:29:38,418 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:29:38,420 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:29:39,685 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:29:44,685 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:29:49,686 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:29:53,417 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:29:53,418 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:29:53,459 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:29:55,642 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:29:56,312 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/config.yaml
+2024-08-23 16:30:00,842 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:30:05,842 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:30:08,417 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:30:08,418 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:30:08,459 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:30:11,641 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:30:16,641 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:30:21,642 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:30:23,378 DEBUG   SystemMonitor:11966 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-08-23 16:30:23,380 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:30:23,417 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:30:23,418 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:30:23,459 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:30:27,617 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:30:28,122 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:30:30,329 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:30:33,164 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:30:38,165 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:30:38,418 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:30:38,418 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:30:38,418 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:30:43,611 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:30:48,611 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:30:53,380 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:30:53,418 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:30:53,418 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:30:53,459 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:30:53,639 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:30:58,640 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:31:03,640 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:31:08,418 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:31:08,418 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:31:08,459 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:31:08,666 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:31:13,666 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:31:18,667 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:31:23,382 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:31:23,418 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:31:23,418 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:31:23,459 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:31:23,682 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:31:23,901 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:31:23,904 DEBUG   SenderThread:11966 [sender.py:send():382] send: history
+2024-08-23 16:31:23,904 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:31:23,906 INFO    SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:31:24,355 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json
+2024-08-23 16:31:26,356 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:31:28,944 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:31:33,945 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:31:38,418 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:31:38,418 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:31:38,419 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:31:39,638 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:31:44,639 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:31:49,639 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:31:53,383 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:31:53,418 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:31:53,418 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:31:53,459 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:31:54,679 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:31:59,679 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:32:04,680 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:32:08,418 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:32:08,418 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:32:08,459 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:32:10,613 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:32:15,613 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:32:19,715 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:32:19,717 DEBUG   SenderThread:11966 [sender.py:send():382] send: history
+2024-08-23 16:32:19,717 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:32:19,719 INFO    SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:32:20,383 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json
+2024-08-23 16:32:20,760 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:32:22,384 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:32:23,385 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:32:23,418 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:32:23,419 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:32:23,420 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:32:26,593 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:32:31,594 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:32:36,594 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:32:38,418 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:32:38,419 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:32:38,459 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:32:41,674 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:32:46,675 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:32:51,676 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:32:53,384 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:32:53,418 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:32:53,419 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:32:53,459 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:32:57,603 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:33:02,604 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:33:07,604 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:33:08,418 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:33:08,419 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:33:08,459 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:33:12,676 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:33:15,428 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:33:15,430 DEBUG   SenderThread:11966 [sender.py:send():382] send: history
+2024-08-23 16:33:15,430 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:33:15,432 INFO    SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:33:16,412 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json
+2024-08-23 16:33:18,413 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:33:18,472 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:33:23,385 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:33:23,419 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:33:23,419 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:33:23,420 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:33:23,682 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:33:28,682 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:33:33,683 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:33:38,419 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:33:38,419 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:33:38,463 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:33:39,667 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:33:44,667 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:33:49,668 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:33:53,386 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:33:53,419 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:33:53,419 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:33:53,463 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:33:55,611 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:34:00,612 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:34:05,612 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:34:08,419 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:34:08,419 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:34:08,463 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:34:10,679 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:34:11,150 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:34:11,152 DEBUG   SenderThread:11966 [sender.py:send():382] send: history
+2024-08-23 16:34:11,152 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:34:11,154 INFO    SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:34:11,439 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json
+2024-08-23 16:34:12,439 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:34:16,192 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:34:21,193 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:34:23,387 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:34:23,419 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:34:23,419 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:34:23,420 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:34:26,589 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:34:31,590 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:34:36,590 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:34:38,419 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:34:38,419 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:34:38,463 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:34:41,674 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:34:46,674 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:34:51,674 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:34:53,389 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:34:53,419 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:34:53,419 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:34:53,463 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:34:57,671 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:35:02,672 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:35:06,945 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:35:06,948 DEBUG   SenderThread:11966 [sender.py:send():382] send: history
+2024-08-23 16:35:06,948 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:35:06,949 INFO    SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:35:07,467 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json
+2024-08-23 16:35:07,988 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:35:08,419 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:35:08,419 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:35:08,421 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:35:08,468 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:35:13,689 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:35:18,689 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:35:23,389 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:35:23,419 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:35:23,419 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:35:23,463 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:35:24,596 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:35:29,597 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:35:34,597 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:35:38,419 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:35:38,420 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:35:38,463 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:35:39,689 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:35:44,690 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:35:49,690 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:35:53,390 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:35:53,419 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:35:53,420 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:35:53,463 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:35:55,688 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:36:00,688 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:36:02,665 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:36:02,667 DEBUG   SenderThread:11966 [sender.py:send():382] send: history
+2024-08-23 16:36:02,667 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:36:02,668 INFO    SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:36:03,495 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json
+2024-08-23 16:36:04,496 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:36:05,708 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:36:08,420 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:36:08,420 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:36:08,421 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:36:11,641 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:36:16,641 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:36:21,642 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:36:23,391 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:36:23,420 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:36:23,420 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:36:23,463 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:36:26,684 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:36:31,685 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:36:36,685 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:36:38,420 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:36:38,420 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:36:38,463 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:36:41,692 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:36:46,692 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:36:51,693 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:36:53,392 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:36:53,420 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:36:53,420 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:36:53,463 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:36:57,615 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:36:58,466 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: partial_history
+2024-08-23 16:36:58,468 DEBUG   SenderThread:11966 [sender.py:send():382] send: history
+2024-08-23 16:36:58,468 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:36:58,469 INFO    SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:36:58,523 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json
+2024-08-23 16:37:00,524 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:37:03,508 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:37:08,420 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:37:08,420 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:37:08,422 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:37:08,688 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:37:13,688 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:37:18,689 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:37:23,393 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:37:23,420 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:37:23,420 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:37:23,463 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:37:24,586 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:37:29,587 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:37:34,587 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:37:38,420 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: stop_status
+2024-08-23 16:37:38,421 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: stop_status
+2024-08-23 16:37:38,463 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:37:40,196 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:37:45,197 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:37:48,676 DEBUG   SenderThread:11966 [sender.py:send():382] send: exit
+2024-08-23 16:37:48,676 INFO    SenderThread:11966 [sender.py:send_exit():589] handling exit code: 255
+2024-08-23 16:37:48,676 INFO    SenderThread:11966 [sender.py:send_exit():591] handling runtime: 505
+2024-08-23 16:37:48,677 INFO    SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:37:48,677 INFO    SenderThread:11966 [sender.py:send_exit():597] send defer
+2024-08-23 16:37:48,678 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:48,678 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 0
+2024-08-23 16:37:48,678 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:48,678 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 0
+2024-08-23 16:37:48,678 INFO    SenderThread:11966 [sender.py:transition_state():617] send defer: 1
+2024-08-23 16:37:48,678 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:48,678 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 1
+2024-08-23 16:37:48,678 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:48,678 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 1
+2024-08-23 16:37:48,678 INFO    SenderThread:11966 [sender.py:transition_state():617] send defer: 2
+2024-08-23 16:37:48,678 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:48,678 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 2
+2024-08-23 16:37:48,678 INFO    HandlerThread:11966 [system_monitor.py:finish():203] Stopping system monitor
+2024-08-23 16:37:48,679 DEBUG   SystemMonitor:11966 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-08-23 16:37:48,679 INFO    HandlerThread:11966 [interfaces.py:finish():202] Joined cpu monitor
+2024-08-23 16:37:48,679 DEBUG   SystemMonitor:11966 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-08-23 16:37:48,679 INFO    HandlerThread:11966 [interfaces.py:finish():202] Joined disk monitor
+2024-08-23 16:37:48,712 INFO    HandlerThread:11966 [interfaces.py:finish():202] Joined gpu monitor
+2024-08-23 16:37:48,712 INFO    HandlerThread:11966 [interfaces.py:finish():202] Joined memory monitor
+2024-08-23 16:37:48,712 INFO    HandlerThread:11966 [interfaces.py:finish():202] Joined network monitor
+2024-08-23 16:37:48,713 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:48,713 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 2
+2024-08-23 16:37:48,713 INFO    SenderThread:11966 [sender.py:transition_state():617] send defer: 3
+2024-08-23 16:37:48,713 DEBUG   SenderThread:11966 [sender.py:send():382] send: stats
+2024-08-23 16:37:48,713 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:48,713 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 3
+2024-08-23 16:37:48,715 DEBUG   SenderThread:11966 [sender.py:send():382] send: history
+2024-08-23 16:37:48,715 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: summary_record
+2024-08-23 16:37:48,716 INFO    SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:37:48,716 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:48,716 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 3
+2024-08-23 16:37:48,716 INFO    SenderThread:11966 [sender.py:transition_state():617] send defer: 4
+2024-08-23 16:37:48,716 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:48,716 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 4
+2024-08-23 16:37:48,717 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:48,717 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 4
+2024-08-23 16:37:48,717 INFO    SenderThread:11966 [sender.py:transition_state():617] send defer: 5
+2024-08-23 16:37:48,717 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:48,717 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 5
+2024-08-23 16:37:48,718 DEBUG   SenderThread:11966 [sender.py:send():382] send: summary
+2024-08-23 16:37:48,718 INFO    SenderThread:11966 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-08-23 16:37:48,719 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:48,719 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 5
+2024-08-23 16:37:48,719 INFO    SenderThread:11966 [sender.py:transition_state():617] send defer: 6
+2024-08-23 16:37:48,719 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:48,719 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 6
+2024-08-23 16:37:48,719 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:48,719 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 6
+2024-08-23 16:37:48,719 INFO    SenderThread:11966 [sender.py:transition_state():617] send defer: 7
+2024-08-23 16:37:48,719 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:37:48,719 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:48,719 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 7
+2024-08-23 16:37:48,720 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:48,720 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 7
+2024-08-23 16:37:49,550 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json
+2024-08-23 16:37:49,671 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 16:37:50,524 INFO    SenderThread:11966 [sender.py:transition_state():617] send defer: 8
+2024-08-23 16:37:50,524 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 16:37:50,524 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:50,524 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 8
+2024-08-23 16:37:50,524 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:50,524 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 8
+2024-08-23 16:37:50,525 INFO    SenderThread:11966 [job_builder.py:build():296] Attempting to build job artifact
+2024-08-23 16:37:50,525 INFO    SenderThread:11966 [job_builder.py:_get_source_type():426] is repo sourced job
+2024-08-23 16:37:50,540 INFO    SenderThread:11966 [job_builder.py:build():402] adding wandb-job metadata file
+2024-08-23 16:37:50,549 INFO    SenderThread:11966 [sender.py:transition_state():617] send defer: 9
+2024-08-23 16:37:50,550 DEBUG   SenderThread:11966 [sender.py:send():382] send: artifact
+2024-08-23 16:37:50,550 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:50,551 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 9
+2024-08-23 16:37:50,551 INFO    Thread-12 :11966 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:37:50,671 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 16:37:51,630 INFO    wandb-upload_0:11966 [upload_job.py:push():86] Skipped uploading /singularity_home/.local/share/wandb/artifacts/staging/tmp7k_y5w7r
+2024-08-23 16:37:52,084 INFO    wandb-upload_1:11966 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmp4s29y7vc
+2024-08-23 16:37:53,444 INFO    SenderThread:11966 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE2MjAxODA1Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjQ1ODQ1MA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE2MTk5MDU4OQ==', 'versionIndex': 2}}}
+2024-08-23 16:37:53,444 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:53,444 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 9
+2024-08-23 16:37:53,444 INFO    SenderThread:11966 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-08-23 16:37:53,553 INFO    SenderThread:11966 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240823_162922-z3gs82jm/files
+2024-08-23 16:37:53,553 INFO    SenderThread:11966 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162922-z3gs82jm/files/requirements.txt requirements.txt
+2024-08-23 16:37:53,553 INFO    SenderThread:11966 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162922-z3gs82jm/files/config.yaml config.yaml
+2024-08-23 16:37:53,555 INFO    SenderThread:11966 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-metadata.json wandb-metadata.json
+2024-08-23 16:37:53,555 INFO    SenderThread:11966 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json wandb-summary.json
+2024-08-23 16:37:53,556 INFO    SenderThread:11966 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_162922-z3gs82jm/files/output.log output.log
+2024-08-23 16:37:53,558 INFO    SenderThread:11966 [sender.py:transition_state():617] send defer: 10
+2024-08-23 16:37:53,558 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 16:37:53,559 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:53,559 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 10
+2024-08-23 16:37:53,560 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:53,560 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 10
+2024-08-23 16:37:53,560 INFO    SenderThread:11966 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-23 16:37:53,672 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 16:37:53,672 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 16:37:53,988 INFO    wandb-upload_0:11966 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162922-z3gs82jm/files/requirements.txt
+2024-08-23 16:37:54,025 INFO    wandb-upload_1:11966 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162922-z3gs82jm/files/config.yaml
+2024-08-23 16:37:54,673 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 16:37:54,673 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 16:37:55,582 INFO    wandb-upload_2:11966 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162922-z3gs82jm/files/wandb-summary.json
+2024-08-23 16:37:55,638 INFO    wandb-upload_3:11966 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_162922-z3gs82jm/files/output.log
+2024-08-23 16:37:55,673 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 16:37:55,674 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 16:37:55,674 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: status_report
+2024-08-23 16:37:55,838 INFO    Thread-11 (_thread_body):11966 [sender.py:transition_state():617] send defer: 11
+2024-08-23 16:37:55,839 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:55,839 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 11
+2024-08-23 16:37:55,839 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:55,839 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 11
+2024-08-23 16:37:55,839 INFO    SenderThread:11966 [file_pusher.py:join():178] waiting for file pusher
+2024-08-23 16:37:55,840 INFO    SenderThread:11966 [sender.py:transition_state():617] send defer: 12
+2024-08-23 16:37:55,840 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:55,840 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 12
+2024-08-23 16:37:55,840 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:55,840 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 12
+2024-08-23 16:37:55,840 INFO    SenderThread:11966 [file_stream.py:finish():595] file stream finish called
+2024-08-23 16:37:56,021 INFO    SenderThread:11966 [file_stream.py:finish():599] file stream finish is done
+2024-08-23 16:37:56,021 INFO    SenderThread:11966 [sender.py:transition_state():617] send defer: 13
+2024-08-23 16:37:56,021 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:56,021 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 13
+2024-08-23 16:37:56,021 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:56,021 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 13
+2024-08-23 16:37:56,021 INFO    SenderThread:11966 [sender.py:transition_state():617] send defer: 14
+2024-08-23 16:37:56,021 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: defer
+2024-08-23 16:37:56,021 DEBUG   SenderThread:11966 [sender.py:send():382] send: final
+2024-08-23 16:37:56,021 INFO    HandlerThread:11966 [handler.py:handle_request_defer():172] handle defer: 14
+2024-08-23 16:37:56,022 DEBUG   SenderThread:11966 [sender.py:send():382] send: footer
+2024-08-23 16:37:56,022 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: defer
+2024-08-23 16:37:56,022 INFO    SenderThread:11966 [sender.py:send_request_defer():613] handle sender defer: 14
+2024-08-23 16:37:56,022 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 16:37:56,022 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 16:37:56,023 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: poll_exit
+2024-08-23 16:37:56,023 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: poll_exit
+2024-08-23 16:37:56,023 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: server_info
+2024-08-23 16:37:56,023 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: get_summary
+2024-08-23 16:37:56,024 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: server_info
+2024-08-23 16:37:56,025 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: sampled_history
+2024-08-23 16:37:56,026 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: internal_messages
+2024-08-23 16:37:56,027 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: job_info
+2024-08-23 16:37:56,184 DEBUG   SenderThread:11966 [sender.py:send_request():409] send_request: job_info
+2024-08-23 16:37:56,185 INFO    MainThread:11966 [wandb_run.py:_footer_history_summary_info():3866] rendering history
+2024-08-23 16:37:56,185 INFO    MainThread:11966 [wandb_run.py:_footer_history_summary_info():3898] rendering summary
+2024-08-23 16:37:56,186 INFO    MainThread:11966 [wandb_run.py:_footer_sync_info():3825] logging synced files
+2024-08-23 16:37:56,186 DEBUG   HandlerThread:11966 [handler.py:handle_request():146] handle_request: shutdown
+2024-08-23 16:37:56,186 INFO    HandlerThread:11966 [handler.py:finish():869] shutting down handler
+2024-08-23 16:37:57,027 INFO    WriterThread:11966 [datastore.py:close():296] close: /project/wandb/run-20240823_162922-z3gs82jm/run-z3gs82jm.wandb
+2024-08-23 16:37:57,185 INFO    SenderThread:11966 [sender.py:finish():1572] shutting down sender
+2024-08-23 16:37:57,185 INFO    SenderThread:11966 [file_pusher.py:finish():172] shutting down file pusher
+2024-08-23 16:37:57,185 INFO    SenderThread:11966 [file_pusher.py:join():178] waiting for file pusher